165 lines
7.3 KiB
Bash
Executable File
165 lines
7.3 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# Main pipeline orchestrator.
|
|
#
|
|
# Execution order:
|
|
# 1. Create cold archives for the selected Docker Compose projects.
|
|
# 2. Run each enabled transfer stage listed in BACKUP_METHODS.
|
|
# 3. Prune old local archives only after every earlier stage succeeds.
|
|
#
|
|
# An EXIT trap publishes the final MQTT result even when a stage fails. The
|
|
# original exit code is preserved so cron, systemd and shell callers can detect
|
|
# failure normally.
|
|
set -Eeuo pipefail
|
|
SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd -P)"
|
|
. "$SCRIPT_DIR/lib/common.sh"
|
|
DRY_RUN=0; SKIP_CONTAINERS=0; SKIP_PRUNE=0; CONTAINERS_OVERRIDE=""
|
|
while [[ $# -gt 0 ]]; do
|
|
case "$1" in
|
|
--dry-run|-t) DRY_RUN=1 ;; --skip-containers) SKIP_CONTAINERS=1 ;; --skip-prune) SKIP_PRUNE=1 ;;
|
|
--containers|-f) CONTAINERS_OVERRIDE="${2:?Missing containers file}"; shift ;;
|
|
-h|--help)
|
|
echo "Usage: ${0##*/} [--dry-run] [--containers FILE] [--skip-containers] [--skip-prune]"
|
|
echo "Transfer stages run in BACKUP_METHODS order. Add stages/NN-name.sh and list name in BACKUP_METHODS."
|
|
exit 0 ;;
|
|
*) die "Unknown option: $1" 2 ;;
|
|
esac
|
|
shift
|
|
done
|
|
|
|
if (( SKIP_CONTAINERS )); then
|
|
BACKUP_LOCAL_ARCHIVE_STATUS="skipped"
|
|
else
|
|
BACKUP_LOCAL_ARCHIVE_STATUS="running"
|
|
fi
|
|
BACKUP_RCLONE_STATUS="not_configured"
|
|
BACKUP_SMB_STATUS="not_configured"
|
|
for configured_method in $BACKUP_METHODS; do
|
|
case "$configured_method" in
|
|
rclone) if is_true "$RCLONE_ENABLED"; then BACKUP_RCLONE_STATUS="pending"; else BACKUP_RCLONE_STATUS="skipped"; fi ;;
|
|
smb) if is_true "$SMB_ENABLED"; then BACKUP_SMB_STATUS="pending"; else BACKUP_SMB_STATUS="skipped"; fi ;;
|
|
esac
|
|
done
|
|
BACKUP_REMOTE_COPY_STATUS="pending"
|
|
if [[ "$BACKUP_RCLONE_STATUS" == "not_configured" || "$BACKUP_RCLONE_STATUS" == "skipped" ]] && [[ "$BACKUP_SMB_STATUS" == "not_configured" || "$BACKUP_SMB_STATUS" == "skipped" ]]; then
|
|
BACKUP_REMOTE_COPY_STATUS="skipped"
|
|
fi
|
|
BACKUP_REMOTE_COPY_FINISHED_AT=""
|
|
export BACKUP_LOCAL_ARCHIVE_STATUS BACKUP_REMOTE_COPY_STATUS BACKUP_RCLONE_STATUS BACKUP_SMB_STATUS BACKUP_REMOTE_COPY_FINISHED_AT
|
|
|
|
BACKUP_RUN_ID="$(run_id)"
|
|
BACKUP_STARTED_EPOCH="$(date +%s)"
|
|
# Every script sourced through this process appends structured log lines here.
|
|
# The MQTT reporting stage parses this temporary log to build its summary.
|
|
BACKUP_RUN_LOG="$(mktemp "${TMPDIR:-/tmp}/zorruno-docker-backup.XXXXXX.log")"
|
|
export BACKUP_RUN_ID BACKUP_STARTED_EPOCH BACKUP_RUN_LOG
|
|
|
|
finish_run() {
|
|
local exit_code=$? finished status archived_count remote_success=0 remote_failure=0 remote_running=0 remote_dry_run=0 transfer_status
|
|
# Prevent recursion when this function exits with the pipeline's status.
|
|
trap - EXIT
|
|
# Reporting is best-effort and must never replace the real pipeline result.
|
|
set +e
|
|
finished="$(date +%s)"
|
|
if [[ "$BACKUP_LOCAL_ARCHIVE_STATUS" == "running" ]]; then
|
|
if (( DRY_RUN )); then
|
|
BACKUP_LOCAL_ARCHIVE_STATUS="dry_run"
|
|
elif (( exit_code == 0 )); then
|
|
BACKUP_LOCAL_ARCHIVE_STATUS="success"
|
|
else
|
|
archived_count="$(grep -c "\[ARCHIVE\] Created " "$BACKUP_RUN_LOG" || true)"
|
|
if (( archived_count > 0 )); then BACKUP_LOCAL_ARCHIVE_STATUS="partial"; else BACKUP_LOCAL_ARCHIVE_STATUS="failed"; fi
|
|
fi
|
|
fi
|
|
for transfer_status in BACKUP_RCLONE_STATUS BACKUP_SMB_STATUS; do
|
|
if [[ "${!transfer_status}" == "running" ]]; then
|
|
if (( DRY_RUN )); then printf -v "$transfer_status" "%s" "dry_run"; else printf -v "$transfer_status" "%s" "failed"; fi
|
|
elif [[ "${!transfer_status}" == "pending" ]]; then
|
|
printf -v "$transfer_status" "%s" "not_attempted"
|
|
fi
|
|
done
|
|
for transfer_status in "$BACKUP_RCLONE_STATUS" "$BACKUP_SMB_STATUS"; do
|
|
case "$transfer_status" in
|
|
success) (( remote_success++ )) ;;
|
|
partial) (( remote_success++ )); (( remote_failure++ )) ;;
|
|
failed) (( remote_failure++ )) ;;
|
|
running|pending) (( remote_running++ )) ;;
|
|
dry_run) (( remote_dry_run++ )) ;;
|
|
esac
|
|
done
|
|
if (( remote_running > 0 )); then
|
|
BACKUP_REMOTE_COPY_STATUS="running"
|
|
elif (( remote_failure > 0 && remote_success > 0 )); then
|
|
BACKUP_REMOTE_COPY_STATUS="partial"
|
|
elif (( remote_failure > 0 )); then
|
|
BACKUP_REMOTE_COPY_STATUS="failed"
|
|
elif (( remote_success > 0 )); then
|
|
BACKUP_REMOTE_COPY_STATUS="success"
|
|
elif (( remote_dry_run > 0 )); then
|
|
BACKUP_REMOTE_COPY_STATUS="dry_run"
|
|
else
|
|
BACKUP_REMOTE_COPY_STATUS="skipped"
|
|
fi
|
|
if (( exit_code == 0 )); then
|
|
if (( DRY_RUN )); then status="dry_run_success"; else status="success"; fi
|
|
else
|
|
if (( DRY_RUN )); then status="dry_run_failed"; else status="failed"; fi
|
|
fi
|
|
if is_true "$MQTT_REPORT_ENABLED"; then
|
|
"$SCRIPT_DIR/stages/99-mqtt-report.sh" "$status" "$exit_code" \
|
|
"$BACKUP_STARTED_EPOCH" "$finished" "$BACKUP_RUN_ID" "$BACKUP_RUN_LOG" \
|
|
|| log ERROR "Unable to publish final MQTT result"
|
|
fi
|
|
rm -f -- "$BACKUP_RUN_LOG"
|
|
exit "$exit_code"
|
|
}
|
|
trap finish_run EXIT
|
|
|
|
require_command flock
|
|
mkdir -p "$(dirname "$LOCK_FILE")"
|
|
# Keep descriptor 9 open for the whole run. Closing it automatically releases
|
|
# the advisory lock, including on errors and signals that terminate the shell.
|
|
exec 9>"$LOCK_FILE"
|
|
flock -n 9 || die "Another backup run holds lock: $LOCK_FILE"
|
|
stage_args=()
|
|
(( DRY_RUN )) && stage_args+=(--dry-run)
|
|
[[ -n "$CONTAINERS_OVERRIDE" ]] && stage_args+=(--containers "$CONTAINERS_OVERRIDE")
|
|
log START "Backup pipeline for $BACKUP_HOSTNAME"
|
|
if is_true "$MQTT_REPORT_ENABLED"; then
|
|
# The retained running state lets Home Assistant show that work is active.
|
|
"$SCRIPT_DIR/stages/99-mqtt-report.sh" running 0 "$BACKUP_STARTED_EPOCH" \
|
|
"$BACKUP_STARTED_EPOCH" "$BACKUP_RUN_ID" "$BACKUP_RUN_LOG" \
|
|
|| log WARN "Unable to publish MQTT running state; backup will continue"
|
|
fi
|
|
if (( SKIP_CONTAINERS )); then
|
|
BACKUP_LOCAL_ARCHIVE_STATUS="skipped"
|
|
else
|
|
"$SCRIPT_DIR/stages/10-containers.sh" "${stage_args[@]}"
|
|
if (( DRY_RUN )); then BACKUP_LOCAL_ARCHIVE_STATUS="dry_run"; else BACKUP_LOCAL_ARCHIVE_STATUS="success"; fi
|
|
fi
|
|
method_args=(); (( DRY_RUN )) && method_args+=(--dry-run)
|
|
# BACKUP_METHODS contains short names (for example: "rclone smb"). Numeric
|
|
# prefixes control directory ordering without becoming part of configuration.
|
|
for method in $BACKUP_METHODS; do
|
|
method_script=""
|
|
while IFS= read -r candidate; do method_script="$candidate"; break; done < <(find "$SCRIPT_DIR/stages" -maxdepth 1 -type f -name "*-${method}.sh" -print | sort)
|
|
[[ -n "$method_script" ]] || die "Configured backup method not found: $method"
|
|
case "$method" in
|
|
rclone) if is_true "$RCLONE_ENABLED"; then BACKUP_RCLONE_STATUS="running"; fi ;;
|
|
smb) if is_true "$SMB_ENABLED"; then BACKUP_SMB_STATUS="running"; fi ;;
|
|
esac
|
|
"$method_script" "${method_args[@]}"
|
|
if (( DRY_RUN )); then
|
|
case "$method" in
|
|
rclone) [[ "$BACKUP_RCLONE_STATUS" == "running" ]] && BACKUP_RCLONE_STATUS="dry_run" ;;
|
|
smb) [[ "$BACKUP_SMB_STATUS" == "running" ]] && BACKUP_SMB_STATUS="dry_run" ;;
|
|
esac
|
|
else
|
|
case "$method" in
|
|
rclone) if [[ "$BACKUP_RCLONE_STATUS" == "running" ]]; then BACKUP_RCLONE_STATUS="success"; BACKUP_REMOTE_COPY_FINISHED_AT="$(date -u +%FT%TZ)"; fi ;;
|
|
smb) if [[ "$BACKUP_SMB_STATUS" == "running" ]]; then BACKUP_SMB_STATUS="success"; BACKUP_REMOTE_COPY_FINISHED_AT="$(date -u +%FT%TZ)"; fi ;;
|
|
esac
|
|
fi
|
|
done
|
|
(( SKIP_PRUNE )) || "$SCRIPT_DIR/stages/90-prune.sh" "${method_args[@]}"
|
|
log DONE "Backup pipeline completed successfully"
|