From a61e962ab5ec084ffa028439a9c8c026fd001bef Mon Sep 17 00:00:00 2001 From: howard Date: Wed, 14 Oct 2026 15:46:13 +1300 Subject: [PATCH] Track current backup pipeline and nested project handling --- README.md | 3 ++ backup.sh | 83 +++++++++++++++++++++++++++++++++++++++- stages/10-containers.sh | 63 ++++++++++++++++++++++++++++-- stages/99-mqtt-report.sh | 10 ++++- 4 files changed, 152 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index a30e25a..fd22602 100644 --- a/README.md +++ b/README.md @@ -57,6 +57,9 @@ container name or a directory name beneath `/dockervolumes`. Blank lines and text after `#` are ignored. If several entries resolve to the same Compose project directory, that project is backed up only once. Keeping this host-local file out of Git prevents a pull on any server from replacing its local list. +When a listed directory contains other running Compose projects, the archive +stage also stops those nested projects before archiving and starts them again +afterward. Intentionally stopped nested projects are left stopped. ## Configuration diff --git a/backup.sh b/backup.sh index a2b6fa0..cadfdff 100755 --- a/backup.sh +++ b/backup.sh @@ -26,6 +26,26 @@ while [[ $# -gt 0 ]]; do shift done +if (( SKIP_CONTAINERS )); then + BACKUP_LOCAL_ARCHIVE_STATUS="skipped" +else + BACKUP_LOCAL_ARCHIVE_STATUS="running" +fi +BACKUP_RCLONE_STATUS="not_configured" +BACKUP_SMB_STATUS="not_configured" +for configured_method in $BACKUP_METHODS; do + case "$configured_method" in + rclone) if is_true "$RCLONE_ENABLED"; then BACKUP_RCLONE_STATUS="pending"; else BACKUP_RCLONE_STATUS="skipped"; fi ;; + smb) if is_true "$SMB_ENABLED"; then BACKUP_SMB_STATUS="pending"; else BACKUP_SMB_STATUS="skipped"; fi ;; + esac +done +BACKUP_REMOTE_COPY_STATUS="pending" +if [[ "$BACKUP_RCLONE_STATUS" == "not_configured" || "$BACKUP_RCLONE_STATUS" == "skipped" ]] && [[ "$BACKUP_SMB_STATUS" == "not_configured" || "$BACKUP_SMB_STATUS" == "skipped" ]]; then + BACKUP_REMOTE_COPY_STATUS="skipped" +fi +BACKUP_REMOTE_COPY_FINISHED_AT="" +export BACKUP_LOCAL_ARCHIVE_STATUS BACKUP_REMOTE_COPY_STATUS BACKUP_RCLONE_STATUS BACKUP_SMB_STATUS BACKUP_REMOTE_COPY_FINISHED_AT + BACKUP_RUN_ID="$(run_id)" BACKUP_STARTED_EPOCH="$(date +%s)" # Every script sourced through this process appends structured log lines here. @@ -34,12 +54,51 @@ BACKUP_RUN_LOG="$(mktemp "${TMPDIR:-/tmp}/zorruno-docker-backup.XXXXXX.log")" export BACKUP_RUN_ID BACKUP_STARTED_EPOCH BACKUP_RUN_LOG finish_run() { - local exit_code=$? finished status + local exit_code=$? finished status archived_count remote_success=0 remote_failure=0 remote_running=0 remote_dry_run=0 transfer_status # Prevent recursion when this function exits with the pipeline's status. trap - EXIT # Reporting is best-effort and must never replace the real pipeline result. set +e finished="$(date +%s)" + if [[ "$BACKUP_LOCAL_ARCHIVE_STATUS" == "running" ]]; then + if (( DRY_RUN )); then + BACKUP_LOCAL_ARCHIVE_STATUS="dry_run" + elif (( exit_code == 0 )); then + BACKUP_LOCAL_ARCHIVE_STATUS="success" + else + archived_count="$(grep -c "\[ARCHIVE\] Created " "$BACKUP_RUN_LOG" || true)" + if (( archived_count > 0 )); then BACKUP_LOCAL_ARCHIVE_STATUS="partial"; else BACKUP_LOCAL_ARCHIVE_STATUS="failed"; fi + fi + fi + for transfer_status in BACKUP_RCLONE_STATUS BACKUP_SMB_STATUS; do + if [[ "${!transfer_status}" == "running" ]]; then + if (( DRY_RUN )); then printf -v "$transfer_status" "%s" "dry_run"; else printf -v "$transfer_status" "%s" "failed"; fi + elif [[ "${!transfer_status}" == "pending" ]]; then + printf -v "$transfer_status" "%s" "not_attempted" + fi + done + for transfer_status in "$BACKUP_RCLONE_STATUS" "$BACKUP_SMB_STATUS"; do + case "$transfer_status" in + success) (( remote_success++ )) ;; + partial) (( remote_success++ )); (( remote_failure++ )) ;; + failed) (( remote_failure++ )) ;; + running|pending) (( remote_running++ )) ;; + dry_run) (( remote_dry_run++ )) ;; + esac + done + if (( remote_running > 0 )); then + BACKUP_REMOTE_COPY_STATUS="running" + elif (( remote_failure > 0 && remote_success > 0 )); then + BACKUP_REMOTE_COPY_STATUS="partial" + elif (( remote_failure > 0 )); then + BACKUP_REMOTE_COPY_STATUS="failed" + elif (( remote_success > 0 )); then + BACKUP_REMOTE_COPY_STATUS="success" + elif (( remote_dry_run > 0 )); then + BACKUP_REMOTE_COPY_STATUS="dry_run" + else + BACKUP_REMOTE_COPY_STATUS="skipped" + fi if (( exit_code == 0 )); then if (( DRY_RUN )); then status="dry_run_success"; else status="success"; fi else @@ -71,7 +130,12 @@ if is_true "$MQTT_REPORT_ENABLED"; then "$BACKUP_STARTED_EPOCH" "$BACKUP_RUN_ID" "$BACKUP_RUN_LOG" \ || log WARN "Unable to publish MQTT running state; backup will continue" fi -(( SKIP_CONTAINERS )) || "$SCRIPT_DIR/stages/10-containers.sh" "${stage_args[@]}" +if (( SKIP_CONTAINERS )); then + BACKUP_LOCAL_ARCHIVE_STATUS="skipped" +else + "$SCRIPT_DIR/stages/10-containers.sh" "${stage_args[@]}" + if (( DRY_RUN )); then BACKUP_LOCAL_ARCHIVE_STATUS="dry_run"; else BACKUP_LOCAL_ARCHIVE_STATUS="success"; fi +fi method_args=(); (( DRY_RUN )) && method_args+=(--dry-run) # BACKUP_METHODS contains short names (for example: "rclone smb"). Numeric # prefixes control directory ordering without becoming part of configuration. @@ -79,7 +143,22 @@ for method in $BACKUP_METHODS; do method_script="" while IFS= read -r candidate; do method_script="$candidate"; break; done < <(find "$SCRIPT_DIR/stages" -maxdepth 1 -type f -name "*-${method}.sh" -print | sort) [[ -n "$method_script" ]] || die "Configured backup method not found: $method" + case "$method" in + rclone) if is_true "$RCLONE_ENABLED"; then BACKUP_RCLONE_STATUS="running"; fi ;; + smb) if is_true "$SMB_ENABLED"; then BACKUP_SMB_STATUS="running"; fi ;; + esac "$method_script" "${method_args[@]}" + if (( DRY_RUN )); then + case "$method" in + rclone) [[ "$BACKUP_RCLONE_STATUS" == "running" ]] && BACKUP_RCLONE_STATUS="dry_run" ;; + smb) [[ "$BACKUP_SMB_STATUS" == "running" ]] && BACKUP_SMB_STATUS="dry_run" ;; + esac + else + case "$method" in + rclone) if [[ "$BACKUP_RCLONE_STATUS" == "running" ]]; then BACKUP_RCLONE_STATUS="success"; BACKUP_REMOTE_COPY_FINISHED_AT="$(date -u +%FT%TZ)"; fi ;; + smb) if [[ "$BACKUP_SMB_STATUS" == "running" ]]; then BACKUP_SMB_STATUS="success"; BACKUP_REMOTE_COPY_FINISHED_AT="$(date -u +%FT%TZ)"; fi ;; + esac + fi done (( SKIP_PRUNE )) || "$SCRIPT_DIR/stages/90-prune.sh" "${method_args[@]}" log DONE "Backup pipeline completed successfully" diff --git a/stages/10-containers.sh b/stages/10-containers.sh index 7c467ae..7cac8a7 100755 --- a/stages/10-containers.sh +++ b/stages/10-containers.sh @@ -1,5 +1,6 @@ #!/usr/bin/env bash -# Stage 10: create a cold, compressed archive of each selected Compose project. +# Stage 10: create a cold, compressed archive of each selected Compose project +# directory, including any nested Compose projects. # # Entries may be Compose project-directory names or live container names. A # container name is resolved through its Compose labels. Duplicate entries that @@ -85,9 +86,41 @@ wait_for_project() { done } +discover_child_projects() { + local root="$1" id row project working_dir + local -A seen=() + # Discover live Compose projects by their container labels. Only running + # projects are stopped/restarted, preserving intentionally stopped stacks. + while IFS= read -r id; do + [[ -n "$id" ]] || continue + row="$(docker inspect -f '{{printf "%s\t%s" (index .Config.Labels "com.docker.compose.project") (index .Config.Labels "com.docker.compose.project.working_dir")}}' "$id" 2>/dev/null || true)" + IFS=$'\t' read -r project working_dir <<<"$row" + [[ -n "$project" && -n "$working_dir" ]] || continue + [[ "$working_dir" == "$root"/* ]] || continue + [[ -n "${seen[$working_dir]:-}" ]] && continue + seen[$working_dir]=1 + printf '%s\t%s\n' "$project" "$working_dir" + done < <(docker ps -q) +} + +restart_children() { + local -a projects=("$@") + local record child_project child_dir failed=0 + for record in "${projects[@]}"; do + IFS=$'\t' read -r child_project child_dir <<<"$record" + log INFO "Restarting nested project: $child_project ($child_dir)" + if ! (cd "$child_dir" && compose up -d); then + log ERROR "Unable to restart nested project $child_project" + failed=1 + fi + done + return "$failed" +} + backup_project() { - local project="$1" working_dir="$2" parent base output_dir output attempt + local project="$1" working_dir="$2" parent base output_dir output attempt child_project child_dir record local -a zstd_args=() + local -a child_records=() stopped_children=() parent="$(dirname "$working_dir")" base="$(basename "$working_dir")" output_dir="$BACKUP_ROOT/$project" @@ -95,14 +128,34 @@ backup_project() { log INFO "Project: $project ($working_dir)" log INFO "Archive: $output" if (( DRY_RUN )); then - log DRY "Would stop, archive, optionally pull, start, and verify $project" + mapfile -t child_records < <(discover_child_projects "$working_dir") + log DRY "Would stop ${#child_records[@]} nested project(s), archive, then restart them with $project" return 0 fi mkdir -p "$output_dir" + mapfile -t child_records < <(discover_child_projects "$working_dir") + for record in "${child_records[@]}"; do + IFS=$'\t' read -r child_project child_dir <<<"$record" + log INFO "Stopping nested project: $child_project ($child_dir)" + # Track before down: Compose may partially stop a project and then fail. + stopped_children+=("$record") + if (cd "$child_dir" && compose down); then + : + else + log ERROR "Unable to stop nested project $child_project" + restart_children "${stopped_children[@]}" || true + return 1 + fi + done cd "$working_dir" # Stopping the whole project creates a consistent filesystem-level backup. - compose down + if ! compose down; then + log ERROR "Unable to stop project $project" + compose up -d || log ERROR "Unable to restart project $project" + restart_children "${stopped_children[@]}" || true + return 1 + fi read -r -a zstd_args <<<"$ZSTD_EXTRA_OPTS" if ! tar --one-file-system -cf - -C "$parent" "$base" | zstd "${zstd_args[@]}" -o "$output"; then # Never leave a partial archive that a transfer stage could mistake for a @@ -110,6 +163,7 @@ backup_project() { rm -f -- "$output" log ERROR "Archive failed; attempting to restart $project" compose up -d || true + restart_children "${stopped_children[@]}" || true return 1 fi log ARCHIVE "Created $output" @@ -119,6 +173,7 @@ backup_project() { compose pull || log WARN "Image pull failed for $project; starting with available images" fi compose up -d + restart_children "${stopped_children[@]}" || return 1 if wait_for_project "$project"; then log OK "$project is running and healthy"; return 0; fi for (( attempt=1; attempt<=START_RETRIES; attempt++ )); do diff --git a/stages/99-mqtt-report.sh b/stages/99-mqtt-report.sh index fba03fa..2fb8146 100755 --- a/stages/99-mqtt-report.sh +++ b/stages/99-mqtt-report.sh @@ -12,6 +12,11 @@ STARTED_EPOCH="${3:-$(date +%s)}" FINISHED_EPOCH="${4:-$(date +%s)}" RUN_ID="${5:-unknown}" LOG_FILE="${6:-/dev/null}" +LOCAL_ARCHIVE_STATUS="${BACKUP_LOCAL_ARCHIVE_STATUS:-unknown}" +REMOTE_COPY_STATUS="${BACKUP_REMOTE_COPY_STATUS:-unknown}" +RCLONE_STATUS="${BACKUP_RCLONE_STATUS:-unknown}" +SMB_STATUS="${BACKUP_SMB_STATUS:-unknown}" +REMOTE_COPY_FINISHED_AT="${BACKUP_REMOTE_COPY_FINISHED_AT:-}" require_command jq total=0; archived=0; dry_run=0; error_count=0 @@ -39,7 +44,10 @@ payload="$(jq -cn \ --argjson finished "$FINISHED_EPOCH" --argjson duration "$((FINISHED_EPOCH - STARTED_EPOCH))" \ --argjson total "$total" --argjson archived "$archived" --argjson dry_run "$dry_run" \ --argjson error_count "$error_count" --argjson errors "$errors_json" \ - '{host:$host,status:$status,run_id:$run_id,exit_code:$exit_code, + --arg local_archive_status "$LOCAL_ARCHIVE_STATUS" --arg remote_copy_status "$REMOTE_COPY_STATUS" \ + --arg rclone_status "$RCLONE_STATUS" --arg smb_status "$SMB_STATUS" \ + --arg remote_copy_finished_at "$REMOTE_COPY_FINISHED_AT" \ + '{host:$host,status:$status,local_archive_status:$local_archive_status,remote_copy_status:$remote_copy_status,rclone_status:$rclone_status,smb_status:$smb_status,remote_copy_finished_at:(if $remote_copy_finished_at == "" then null else $remote_copy_finished_at end),run_id:$run_id,exit_code:$exit_code, started_at:($started|todateiso8601),finished_at:($finished|todateiso8601), duration_seconds:$duration,containers_total:$total,containers_backed_up:$archived, containers_dry_run:$dry_run,error_count:$error_count,errors:$errors}')"