379 lines
10 KiB
Bash
379 lines
10 KiB
Bash
#!/usr/bin/env bash
|
|
# ORIGINAL AUTHOR: kmw (updates by zorruno)
|
|
# HISTORY:
|
|
# 2026-04-20 - add configurable SOURCE_BASE_DIR and -s option for source path fallback.
|
|
# 2025-08-22 - add start wait with one retry; fix find_working_dir typo; tidy.
|
|
# 2025-08-22 - tidy, stricter bash, compose detection, better checks, clearer logging.
|
|
# 2022-03-25 - add -t option for dry-run (no backups performed).
|
|
# 2022-01-15 - fix error reporting when a container is not found.
|
|
# 2022-01-09 - ignore trailing comments
|
|
# 2022-01-03 - display help; list from file; -f option
|
|
# 2021-10-16 - structure and logging
|
|
# 2020-05-ish - first version
|
|
|
|
set -Eeuo pipefail
|
|
|
|
timestamp_format='%Y-%m-%d_%H-%M-%S'
|
|
|
|
# verbosity: set non-empty to enable debug
|
|
dbg=
|
|
# dbg=VERBOSE
|
|
|
|
# default containers list file
|
|
containersfile="containers.txt"
|
|
|
|
# dry run (or pass -t)
|
|
dryrun=
|
|
|
|
# base directory for backups
|
|
BACKUP_DIR="/dockerbackups"
|
|
|
|
# base directory for source docker project folders
|
|
SOURCE_BASE_DIR="${SOURCE_BASE_DIR:-/dockervolumes}"
|
|
|
|
# start/health wait tuning
|
|
START_TIMEOUT=90 # seconds to wait for service to be up
|
|
WAIT_INTERVAL=3 # seconds between polls
|
|
RETRIES=1 # number of restart retries on failure (per container)
|
|
|
|
# log file
|
|
logdir="$HOME/.local/share/log/${0##*/}"
|
|
mkdir -p "$logdir"
|
|
ymdhms="$(/bin/date +'%Y-%m-%d_%H:%M:%S')"
|
|
ymd="${ymdhms%%_*}"
|
|
logfile="$logdir/${0##*/}_${ymd}.log"
|
|
|
|
# compose command detection
|
|
COMPOSE_CMD=()
|
|
if docker compose version >/dev/null 2>&1; then
|
|
COMPOSE_CMD=(docker compose)
|
|
elif command -v docker-compose >/dev/null 2>&1; then
|
|
COMPOSE_CMD=(docker-compose)
|
|
else
|
|
echo "ERROR: Neither 'docker compose' nor 'docker-compose' found." >&2
|
|
exit 3
|
|
fi
|
|
|
|
log() {
|
|
local timestamp
|
|
timestamp="$(/bin/date +"$timestamp_format")"
|
|
while [ "$#" -gt 0 ]; do
|
|
echo "$timestamp: ${0##*/}: $1" >>"$logfile"
|
|
if [ -n "$dbg" ]; then
|
|
echo "$timestamp LOG: ${0##*/}: $1" >&2
|
|
else
|
|
echo "$1" >&2
|
|
fi
|
|
shift
|
|
done
|
|
}
|
|
|
|
debug() {
|
|
local timestamp
|
|
timestamp="$(/bin/date +"$timestamp_format")"
|
|
while [ "$#" -gt 0 ]; do
|
|
echo "$timestamp DBG: ${0##*/}: $1" >>"$logfile"
|
|
if [ -n "$dbg" ]; then
|
|
echo "$timestamp DBG: ${0##*/}: $1" >&2
|
|
fi
|
|
shift
|
|
done
|
|
}
|
|
|
|
fail() {
|
|
local exitcode="$1"
|
|
shift
|
|
log "!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!"
|
|
log "${0##*/}: FATAL ERROR"
|
|
while [ "$#" -gt 0 ]; do
|
|
log "$1"
|
|
shift
|
|
done
|
|
sleep 1
|
|
tidy_up
|
|
exit "$exitcode"
|
|
}
|
|
|
|
tidy_up() {
|
|
debug "tidy_up"
|
|
cd || log "Failed to cd in tidy_up"
|
|
trap - EXIT
|
|
trap - INT
|
|
}
|
|
|
|
usage() {
|
|
cat 1>&2 <<EOF
|
|
Usage: ${0##*/} [-f containers_list_file] [-s source_base_dir] [-t] [-v] [-h]
|
|
|
|
-t Test (dry run; no backups performed)
|
|
-v Verbose (debug logging to stderr)
|
|
-f FILE File containing list of container names
|
|
Default: '$containersfile'
|
|
-s DIR Base directory containing container/project folders
|
|
Default: '$SOURCE_BASE_DIR'
|
|
-h Show this help
|
|
|
|
Example:
|
|
${0##*/} -f containers.servername.txt
|
|
${0##*/} -f containers.servername.txt -s /dockervolumes
|
|
EOF
|
|
}
|
|
|
|
parse_cmdline() {
|
|
while getopts ":f:hs:tv" opt; do
|
|
case "$opt" in
|
|
f) containersfile="$OPTARG" ;;
|
|
s) SOURCE_BASE_DIR="$OPTARG" ;;
|
|
h) usage; exit 0 ;;
|
|
t) dryrun=1 ;;
|
|
v) dbg=VERBOSE ;;
|
|
\?) echo "Invalid option: -$OPTARG" 1>&2; usage; exit 1 ;;
|
|
:) echo "Option -$OPTARG requires an argument" 1>&2; exit 1 ;;
|
|
esac
|
|
done
|
|
shift $((OPTIND - 1))
|
|
}
|
|
|
|
toolcheck() {
|
|
local tool
|
|
while [ "$#" -gt 0 ]; do
|
|
tool="$1"
|
|
shift
|
|
debug "Checking for tool: '$tool'"
|
|
if ! command -v "$tool" >/dev/null 2>&1; then
|
|
fail 3 "Failed to find required tool: '$tool'"
|
|
fi
|
|
done
|
|
}
|
|
|
|
compose() {
|
|
"${COMPOSE_CMD[@]}" "$@"
|
|
}
|
|
|
|
# Return "<project> <service>" for a container by inspecting labels
|
|
get_compose_identity() {
|
|
# arg: container_name
|
|
local name="$1"
|
|
local proj serv
|
|
proj="$(docker inspect "$name" 2>/dev/null | jq -r '.[0].Config.Labels["com.docker.compose.project"] // empty')"
|
|
serv="$(docker inspect "$name" 2>/dev/null | jq -r '.[0].Config.Labels["com.docker.compose.service"] // empty')"
|
|
if [ -z "$proj" ] || [ -z "$serv" ]; then
|
|
echo ""
|
|
return 1
|
|
fi
|
|
echo "$proj $serv"
|
|
}
|
|
|
|
# Check current state/health for any container matching project+service
|
|
# Returns 0 if running and healthy (or no healthcheck), else 1
|
|
service_is_healthy() {
|
|
# args: project service
|
|
local proj="$1"
|
|
local serv="$2"
|
|
local ids
|
|
# find containers by labels (may be multiple if scaled; any healthy one counts)
|
|
ids="$(docker ps -a -q --filter "label=com.docker.compose.project=$proj" --filter "label=com.docker.compose.service=$serv")" || true
|
|
if [ -z "$ids" ]; then
|
|
debug "No containers found for project=$proj service=$serv"
|
|
return 1
|
|
fi
|
|
local id
|
|
for id in $ids; do
|
|
# status: created, restarting, running, removing, paused, exited, dead
|
|
local st hlth
|
|
st="$(docker inspect -f '{{.State.Status}}' "$id" 2>/dev/null || echo "unknown")"
|
|
if [ "$st" != "running" ]; then
|
|
continue
|
|
fi
|
|
# Health may not exist; treat missing as pass if running
|
|
hlth="$(docker inspect -f '{{if .State.Health}}{{.State.Health.Status}}{{else}}none{{end}}' "$id" 2>/dev/null || echo "none")"
|
|
if [ "$hlth" = "none" ] || [ "$hlth" = "healthy" ]; then
|
|
debug "Container $id is running (health=$hlth)"
|
|
return 0
|
|
fi
|
|
done
|
|
return 1
|
|
}
|
|
|
|
# Wait for service to become healthy/running within timeout
|
|
wait_for_service() {
|
|
# args: project service timeout_seconds
|
|
local proj="$1"
|
|
local serv="$2"
|
|
local timeout="$3"
|
|
local start now
|
|
start="$(date +%s)"
|
|
while true; do
|
|
if service_is_healthy "$proj" "$serv"; then
|
|
return 0
|
|
fi
|
|
now="$(date +%s)"
|
|
if [ $(( now - start )) -ge "$timeout" ]; then
|
|
return 1
|
|
fi
|
|
sleep "$WAIT_INTERVAL"
|
|
done
|
|
}
|
|
|
|
down_backup_up() {
|
|
# args: WORKING_DIR PROJECT SERVICE
|
|
local WORKING_DIR="$1"
|
|
local PROJ="$2"
|
|
local SERV="$3"
|
|
|
|
local PROJECT_NAME
|
|
local BAK_YMDHMS
|
|
local BACKUP_CODE_DIR
|
|
local BACKUP_CODE_ZSTD
|
|
|
|
PROJECT_NAME="$(basename "$WORKING_DIR")"
|
|
BAK_YMDHMS="$(date +'%Y-%m-%d_%H-%M-%S')"
|
|
BACKUP_CODE_DIR="${BACKUP_DIR}/${PROJECT_NAME}"
|
|
BACKUP_CODE_ZSTD="${BACKUP_CODE_DIR}/$(hostname -s).${PROJECT_NAME}.${BAK_YMDHMS}.zst"
|
|
|
|
mkdir -p "${BACKUP_CODE_DIR}"
|
|
log "Backup target: ${BACKUP_CODE_ZSTD}"
|
|
|
|
if [ -n "$dryrun" ]; then
|
|
log "DRY-RUN: would run: compose down; tar --zstd; compose pull; compose up -d; wait for ${PROJ}/${SERV}"
|
|
return 0
|
|
fi
|
|
|
|
# stop, archive, start, then verify
|
|
compose down || log "WARNING: non-zero exit from 'compose down' (continuing)"
|
|
tar --one-file-system --zstd -cf "${BACKUP_CODE_ZSTD}" "${WORKING_DIR}"
|
|
ls -lh "${BACKUP_CODE_DIR}" || true
|
|
compose pull || log "WARNING: non-zero exit from 'compose pull' (continuing)"
|
|
compose up -d
|
|
|
|
if wait_for_service "$PROJ" "$SERV" "$START_TIMEOUT"; then
|
|
log "Service ${PROJ}/${SERV} is up."
|
|
return 0
|
|
fi
|
|
|
|
log "Service ${PROJ}/${SERV} did not become healthy in ${START_TIMEOUT}s. Retrying ($RETRIES)."
|
|
local attempt=1
|
|
while [ "$attempt" -le "$RETRIES" ]; do
|
|
log "Retry $attempt: restarting ${PROJ}/${SERV}"
|
|
compose down || log "WARNING: retry: non-zero from 'compose down'"
|
|
compose up -d
|
|
if wait_for_service "$PROJ" "$SERV" "$START_TIMEOUT"; then
|
|
log "Service ${PROJ}/${SERV} is up after retry."
|
|
return 0
|
|
fi
|
|
attempt=$(( attempt + 1 ))
|
|
done
|
|
|
|
fail 1 "Service ${PROJ}/${SERV} failed to start/healthy after ${RETRIES} retry(ies)."
|
|
}
|
|
|
|
find_working_dir() {
|
|
# args: CONTAINER_NAME
|
|
local name="$1"
|
|
local wd=""
|
|
if docker inspect "$name" >/dev/null 2>&1; then
|
|
wd="$(docker inspect "$name" 2>/dev/null | jq -r '.[0].Config.Labels["com.docker.compose.project.working_dir"] // empty')"
|
|
fi
|
|
if [ -z "$wd" ]; then
|
|
if [ -d "$SOURCE_BASE_DIR/$name" ]; then
|
|
wd="$SOURCE_BASE_DIR/$name"
|
|
elif [ -d "$HOME/prj/docker/$name" ]; then
|
|
wd="$HOME/prj/docker/$name"
|
|
else
|
|
wd=""
|
|
fi
|
|
fi
|
|
printf '%s' "$wd"
|
|
}
|
|
|
|
container_exists() {
|
|
docker ps -a --format '{{.Names}}' | grep -Fxq "$1"
|
|
}
|
|
|
|
backup_container() {
|
|
local container_name="$1"
|
|
log "Processing container '$container_name'"
|
|
|
|
local container_start_time container_finish_time container_duration container_duration_m
|
|
container_start_time="$(date +%s)"
|
|
|
|
if ! container_exists "$container_name"; then
|
|
log "Container not found: '$container_name' (skipping)"
|
|
return 0
|
|
fi
|
|
|
|
# capture compose identity before we stop anything
|
|
local identity proj serv
|
|
identity="$(get_compose_identity "$container_name" || true)"
|
|
if [ -z "$identity" ]; then
|
|
fail 1 "Could not determine compose identity (project/service) for '$container_name'."
|
|
fi
|
|
proj="$(echo "$identity" | awk '{print $1}')"
|
|
serv="$(echo "$identity" | awk '{print $2}')"
|
|
debug "Compose identity: project=$proj service=$serv"
|
|
|
|
local working_dir
|
|
working_dir="$(find_working_dir "$container_name")"
|
|
if [ -z "$working_dir" ]; then
|
|
fail 1 "Could not determine working_dir for container '$container_name'."
|
|
fi
|
|
log "Working dir: $working_dir"
|
|
if [ ! -d "$working_dir" ]; then
|
|
fail 1 "Working dir does not exist: '$working_dir'"
|
|
fi
|
|
|
|
local original_dir
|
|
original_dir="$(pwd)"
|
|
cd "$working_dir" || fail 1 "Could not cd to '$working_dir'"
|
|
|
|
down_backup_up "$working_dir" "$proj" "$serv" || log "WARNING: non-zero exit from backup function"
|
|
|
|
cd "$original_dir" || fail 1 "Could not cd back to '$original_dir'"
|
|
|
|
container_finish_time="$(date +%s)"
|
|
container_duration=$((container_finish_time - container_start_time))
|
|
container_duration_m=$((container_duration / 60))
|
|
log "Finished '$container_name' after ${container_duration}s (approx ${container_duration_m} min)"
|
|
}
|
|
|
|
trap tidy_up EXIT
|
|
trap 'exit 130' INT
|
|
|
|
parse_cmdline "$@"
|
|
|
|
debug "-----"
|
|
debug "logfile = '$logfile'"
|
|
debug "source_base_dir = '$SOURCE_BASE_DIR'"
|
|
|
|
toolcheck docker jq sed tar
|
|
|
|
if [ ! -f "$containersfile" ]; then
|
|
fail 2 "Containers file not found: '$containersfile'"
|
|
fi
|
|
if [ ! -r "$containersfile" ]; then
|
|
fail 2 "Containers file not readable: '$containersfile'"
|
|
fi
|
|
|
|
containers="$(sed -rn 's/^ +//; s/#.*//; s/ +$//; /^$/b; p' "$containersfile" || true)"
|
|
|
|
if [ -z "${containers:-}" ]; then
|
|
log "No containers listed in '$containersfile' (nothing to do)."
|
|
exit 0
|
|
fi
|
|
|
|
script_start_time="$(date +%s)"
|
|
|
|
for container in $containers; do
|
|
backup_container "$container"
|
|
done
|
|
|
|
script_finish_time="$(date +%s)"
|
|
script_duration=$((script_finish_time - script_start_time))
|
|
script_duration_m=$((script_duration / 60))
|
|
|
|
debug "-----"
|
|
log "Script finished after ${script_duration}s (${script_duration_m} min)"
|
|
|
|
exit 0
|
|
# EOF
|