Initial commit
This commit is contained in:
Executable
+369
@@ -0,0 +1,369 @@
|
||||
#!/usr/bin/env bash
|
||||
# AUTHOR: Karl Mowatt-Wilson (updates by zorruno)
|
||||
# HISTORY:
|
||||
# 2025-08-22 - add start wait with one retry; fix find_working_dir typo; tidy.
|
||||
# 2025-08-22 - tidy, stricter bash, compose detection, better checks, clearer logging.
|
||||
# 2022-03-25 - add -t option for dry-run (no backups performed).
|
||||
# 2022-01-15 - fix error reporting when a container is not found.
|
||||
# 2022-01-09 - ignore trailing comments
|
||||
# 2022-01-03 - display help; list from file; -f option
|
||||
# 2021-10-16 - structure and logging
|
||||
# 2020-05-ish - first version
|
||||
|
||||
set -Eeuo pipefail
|
||||
|
||||
timestamp_format='%Y-%m-%d_%H-%M-%S'
|
||||
|
||||
# verbosity: set non-empty to enable debug
|
||||
dbg=
|
||||
# dbg=VERBOSE
|
||||
|
||||
# default containers list file
|
||||
containersfile="containers.txt"
|
||||
|
||||
# dry run (or pass -t)
|
||||
dryrun=
|
||||
|
||||
# base directory for backups
|
||||
BACKUP_DIR="/dockerbackups"
|
||||
|
||||
# start/health wait tuning
|
||||
START_TIMEOUT=90 # seconds to wait for service to be up
|
||||
WAIT_INTERVAL=3 # seconds between polls
|
||||
RETRIES=1 # number of restart retries on failure (per container)
|
||||
|
||||
# log file
|
||||
logdir="$HOME/.local/share/log/${0##*/}"
|
||||
mkdir -p "$logdir"
|
||||
ymdhms="$(/bin/date +'%Y-%m-%d_%H:%M:%S')"
|
||||
ymd="${ymdhms%%_*}"
|
||||
logfile="$logdir/${0##*/}_${ymd}.log"
|
||||
|
||||
# compose command detection
|
||||
COMPOSE_CMD=()
|
||||
if docker compose version >/dev/null 2>&1; then
|
||||
COMPOSE_CMD=(docker compose)
|
||||
elif command -v docker-compose >/dev/null 2>&1; then
|
||||
COMPOSE_CMD=(docker-compose)
|
||||
else
|
||||
echo "ERROR: Neither 'docker compose' nor 'docker-compose' found." >&2
|
||||
exit 3
|
||||
fi
|
||||
|
||||
log() {
|
||||
local timestamp
|
||||
timestamp="$(/bin/date +"$timestamp_format")"
|
||||
while [ "$#" -gt 0 ]; do
|
||||
echo "$timestamp: ${0##*/}: $1" >>"$logfile"
|
||||
if [ -n "$dbg" ]; then
|
||||
echo "$timestamp LOG: ${0##*/}: $1" >&2
|
||||
else
|
||||
echo "$1" >&2
|
||||
fi
|
||||
shift
|
||||
done
|
||||
}
|
||||
|
||||
debug() {
|
||||
local timestamp
|
||||
timestamp="$(/bin/date +"$timestamp_format")"
|
||||
while [ "$#" -gt 0 ]; do
|
||||
echo "$timestamp DBG: ${0##*/}: $1" >>"$logfile"
|
||||
if [ -n "$dbg" ]; then
|
||||
echo "$timestamp DBG: ${0##*/}: $1" >&2
|
||||
fi
|
||||
shift
|
||||
done
|
||||
}
|
||||
|
||||
fail() {
|
||||
local exitcode="$1"
|
||||
shift
|
||||
log "!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!"
|
||||
log "${0##*/}: FATAL ERROR"
|
||||
while [ "$#" -gt 0 ]; do
|
||||
log "$1"
|
||||
shift
|
||||
done
|
||||
sleep 1
|
||||
tidy_up
|
||||
exit "$exitcode"
|
||||
}
|
||||
|
||||
tidy_up() {
|
||||
debug "tidy_up"
|
||||
cd || log "Failed to cd in tidy_up"
|
||||
trap - EXIT
|
||||
trap - INT
|
||||
}
|
||||
|
||||
usage() {
|
||||
cat 1>&2 <<EOF
|
||||
Usage: ${0##*/} [-f containers_list_file] [-t] [-v] [-h]
|
||||
|
||||
-t Test (dry run; no backups performed)
|
||||
-v Verbose (debug logging to stderr)
|
||||
-f FILE File containing list of container names
|
||||
Default: '$containersfile'
|
||||
-h Show this help
|
||||
|
||||
Example:
|
||||
${0##*/} -f containers.servername.txt
|
||||
EOF
|
||||
}
|
||||
|
||||
parse_cmdline() {
|
||||
while getopts ":f:htv" opt; do
|
||||
case "$opt" in
|
||||
f) containersfile="$OPTARG" ;;
|
||||
h) usage; exit 0 ;;
|
||||
t) dryrun=1 ;;
|
||||
v) dbg=VERBOSE ;;
|
||||
\?) echo "Invalid option: -$OPTARG" 1>&2; usage; exit 1 ;;
|
||||
:) echo "Option -$OPTARG requires an argument" 1>&2; exit 1 ;;
|
||||
esac
|
||||
done
|
||||
shift $((OPTIND - 1))
|
||||
}
|
||||
|
||||
toolcheck() {
|
||||
local tool
|
||||
while [ "$#" -gt 0 ]; do
|
||||
tool="$1"
|
||||
shift
|
||||
debug "Checking for tool: '$tool'"
|
||||
if ! command -v "$tool" >/dev/null 2>&1; then
|
||||
fail 3 "Failed to find required tool: '$tool'"
|
||||
fi
|
||||
done
|
||||
}
|
||||
|
||||
compose() {
|
||||
"${COMPOSE_CMD[@]}" "$@"
|
||||
}
|
||||
|
||||
# Return "<project> <service>" for a container by inspecting labels
|
||||
get_compose_identity() {
|
||||
# arg: container_name
|
||||
local name="$1"
|
||||
local proj serv
|
||||
proj="$(docker inspect "$name" 2>/dev/null | jq -r '.[0].Config.Labels["com.docker.compose.project"] // empty')"
|
||||
serv="$(docker inspect "$name" 2>/dev/null | jq -r '.[0].Config.Labels["com.docker.compose.service"] // empty')"
|
||||
if [ -z "$proj" ] || [ -z "$serv" ]; then
|
||||
echo ""
|
||||
return 1
|
||||
fi
|
||||
echo "$proj $serv"
|
||||
}
|
||||
|
||||
# Check current state/health for any container matching project+service
|
||||
# Returns 0 if running and healthy (or no healthcheck), else 1
|
||||
service_is_healthy() {
|
||||
# args: project service
|
||||
local proj="$1"
|
||||
local serv="$2"
|
||||
local ids
|
||||
# find containers by labels (may be multiple if scaled; any healthy one counts)
|
||||
ids="$(docker ps -a -q --filter "label=com.docker.compose.project=$proj" --filter "label=com.docker.compose.service=$serv")" || true
|
||||
if [ -z "$ids" ]; then
|
||||
debug "No containers found for project=$proj service=$serv"
|
||||
return 1
|
||||
fi
|
||||
local id
|
||||
for id in $ids; do
|
||||
# status: created, restarting, running, removing, paused, exited, dead
|
||||
local st hlth
|
||||
st="$(docker inspect -f '{{.State.Status}}' "$id" 2>/dev/null || echo "unknown")"
|
||||
if [ "$st" != "running" ]; then
|
||||
continue
|
||||
fi
|
||||
# Health may not exist; treat missing as pass if running
|
||||
hlth="$(docker inspect -f '{{if .State.Health}}{{.State.Health.Status}}{{else}}none{{end}}' "$id" 2>/dev/null || echo "none")"
|
||||
if [ "$hlth" = "none" ] || [ "$hlth" = "healthy" ]; then
|
||||
debug "Container $id is running (health=$hlth)"
|
||||
return 0
|
||||
fi
|
||||
done
|
||||
return 1
|
||||
}
|
||||
|
||||
# Wait for service to become healthy/running within timeout
|
||||
wait_for_service() {
|
||||
# args: project service timeout_seconds
|
||||
local proj="$1"
|
||||
local serv="$2"
|
||||
local timeout="$3"
|
||||
local start now
|
||||
start="$(date +%s)"
|
||||
while true; do
|
||||
if service_is_healthy "$proj" "$serv"; then
|
||||
return 0
|
||||
fi
|
||||
now="$(date +%s)"
|
||||
if [ $(( now - start )) -ge "$timeout" ]; then
|
||||
return 1
|
||||
fi
|
||||
sleep "$WAIT_INTERVAL"
|
||||
done
|
||||
}
|
||||
|
||||
down_backup_up() {
|
||||
# args: WORKING_DIR PROJECT SERVICE
|
||||
local WORKING_DIR="$1"
|
||||
local PROJ="$2"
|
||||
local SERV="$3"
|
||||
|
||||
local PROJECT_NAME
|
||||
local BAK_YMDHMS
|
||||
local BACKUP_CODE_DIR
|
||||
local BACKUP_CODE_ZSTD
|
||||
|
||||
PROJECT_NAME="$(basename "$WORKING_DIR")"
|
||||
BAK_YMDHMS="$(date +'%Y-%m-%d_%H-%M-%S')"
|
||||
BACKUP_CODE_DIR="${BACKUP_DIR}/${PROJECT_NAME}"
|
||||
BACKUP_CODE_ZSTD="${BACKUP_CODE_DIR}/$(hostname -s).${PROJECT_NAME}.${BAK_YMDHMS}.zst"
|
||||
|
||||
mkdir -p "${BACKUP_CODE_DIR}"
|
||||
log "Backup target: ${BACKUP_CODE_ZSTD}"
|
||||
|
||||
if [ -n "$dryrun" ]; then
|
||||
log "DRY-RUN: would run: compose down; tar --zstd; compose pull; compose up -d; wait for ${PROJ}/${SERV}"
|
||||
return 0
|
||||
fi
|
||||
|
||||
# stop, archive, start, then verify
|
||||
compose down || log "WARNING: non-zero exit from 'compose down' (continuing)"
|
||||
tar --one-file-system --zstd -cf "${BACKUP_CODE_ZSTD}" "${WORKING_DIR}"
|
||||
ls -lh "${BACKUP_CODE_DIR}" || true
|
||||
compose pull || log "WARNING: non-zero exit from 'compose pull' (continuing)"
|
||||
compose up -d
|
||||
|
||||
if wait_for_service "$PROJ" "$SERV" "$START_TIMEOUT"; then
|
||||
log "Service ${PROJ}/${SERV} is up."
|
||||
return 0
|
||||
fi
|
||||
|
||||
log "Service ${PROJ}/${SERV} did not become healthy in ${START_TIMEOUT}s. Retrying ($RETRIES)."
|
||||
local attempt=1
|
||||
while [ "$attempt" -le "$RETRIES" ]; do
|
||||
log "Retry $attempt: restarting ${PROJ}/${SERV}"
|
||||
compose down || log "WARNING: retry: non-zero from 'compose down'"
|
||||
compose up -d
|
||||
if wait_for_service "$PROJ" "$SERV" "$START_TIMEOUT"; then
|
||||
log "Service ${PROJ}/${SERV} is up after retry."
|
||||
return 0
|
||||
fi
|
||||
attempt=$(( attempt + 1 ))
|
||||
done
|
||||
|
||||
fail 1 "Service ${PROJ}/${SERV} failed to start/healthy after ${RETRIES} retry(ies)."
|
||||
}
|
||||
|
||||
find_working_dir() {
|
||||
# args: CONTAINER_NAME
|
||||
local name="$1"
|
||||
local wd=""
|
||||
if docker inspect "$name" >/dev/null 2>&1; then
|
||||
wd="$(docker inspect "$name" 2>/dev/null | jq -r '.[0].Config.Labels["com.docker.compose.project.working_dir"] // empty')"
|
||||
fi
|
||||
if [ -z "$wd" ]; then
|
||||
if [ -d "/dockervolumes/$name" ]; then
|
||||
wd="/dockervolumes/$name"
|
||||
elif [ -d "$HOME/prj/docker/$name" ]; then
|
||||
wd="$HOME/prj/docker/$name"
|
||||
else
|
||||
wd=""
|
||||
fi
|
||||
fi
|
||||
printf '%s' "$wd"
|
||||
}
|
||||
|
||||
container_exists() {
|
||||
docker ps -a --format '{{.Names}}' | grep -Fxq "$1"
|
||||
}
|
||||
|
||||
backup_container() {
|
||||
local container_name="$1"
|
||||
log "Processing container '$container_name'"
|
||||
|
||||
local container_start_time container_finish_time container_duration container_duration_m
|
||||
container_start_time="$(date +%s)"
|
||||
|
||||
if ! container_exists "$container_name"; then
|
||||
log "Container not found: '$container_name' (skipping)"
|
||||
return 0
|
||||
fi
|
||||
|
||||
# capture compose identity before we stop anything
|
||||
local identity proj serv
|
||||
identity="$(get_compose_identity "$container_name" || true)"
|
||||
if [ -z "$identity" ]; then
|
||||
fail 1 "Could not determine compose identity (project/service) for '$container_name'."
|
||||
fi
|
||||
proj="$(echo "$identity" | awk '{print $1}')"
|
||||
serv="$(echo "$identity" | awk '{print $2}')"
|
||||
debug "Compose identity: project=$proj service=$serv"
|
||||
|
||||
local working_dir
|
||||
working_dir="$(find_working_dir "$container_name")"
|
||||
if [ -z "$working_dir" ]; then
|
||||
fail 1 "Could not determine working_dir for container '$container_name'."
|
||||
fi
|
||||
log "Working dir: $working_dir"
|
||||
if [ ! -d "$working_dir" ]; then
|
||||
fail 1 "Working dir does not exist: '$working_dir'"
|
||||
fi
|
||||
|
||||
local original_dir
|
||||
original_dir="$(pwd)"
|
||||
cd "$working_dir" || fail 1 "Could not cd to '$working_dir'"
|
||||
|
||||
down_backup_up "$working_dir" "$proj" "$serv" || log "WARNING: non-zero exit from backup function"
|
||||
|
||||
cd "$original_dir" || fail 1 "Could not cd back to '$original_dir'"
|
||||
|
||||
container_finish_time="$(date +%s)"
|
||||
container_duration=$((container_finish_time - container_start_time))
|
||||
container_duration_m=$((container_duration / 60))
|
||||
log "Finished '$container_name' after ${container_duration}s (approx ${container_duration_m} min)"
|
||||
}
|
||||
|
||||
trap tidy_up EXIT
|
||||
trap 'exit 130' INT
|
||||
|
||||
parse_cmdline "$@"
|
||||
|
||||
debug "-----"
|
||||
debug "logfile = '$logfile'"
|
||||
|
||||
toolcheck docker jq sed tar
|
||||
|
||||
if [ ! -f "$containersfile" ]; then
|
||||
fail 2 "Containers file not found: '$containersfile'"
|
||||
fi
|
||||
if [ ! -r "$containersfile" ]; then
|
||||
fail 2 "Containers file not readable: '$containersfile'"
|
||||
fi
|
||||
|
||||
containers="$(sed -rn 's/^ +//; s/#.*//; s/ +$//; /^$/b; p' "$containersfile" || true)"
|
||||
|
||||
if [ -z "${containers:-}" ]; then
|
||||
log "No containers listed in '$containersfile' (nothing to do)."
|
||||
exit 0
|
||||
fi
|
||||
|
||||
script_start_time="$(date +%s)"
|
||||
|
||||
for container in $containers; do
|
||||
backup_container "$container"
|
||||
done
|
||||
|
||||
script_finish_time="$(date +%s)"
|
||||
script_duration=$((script_finish_time - script_start_time))
|
||||
script_duration_m=$((script_duration / 60))
|
||||
|
||||
debug "-----"
|
||||
log "Script finished after ${script_duration}s (${script_duration_m} min)"
|
||||
|
||||
exit 0
|
||||
# EOF
|
||||
Reference in New Issue
Block a user