feat: db-tool: normalize-backup
This commit is contained in:
@@ -66,6 +66,28 @@ The `pull_staging` subcommand allows importing data from a remote staging enviro
|
||||
|
||||
If any of these variables are missing when `pull_staging` is invoked, the tool will exit with code 3 and print the name of the missing variable to stderr.
|
||||
|
||||
## normalize-backup Contract
|
||||
|
||||
The `normalize-backup` subcommand converts a physical PostgreSQL backup into a
|
||||
local-development restore artifact without modifying the input backup:
|
||||
|
||||
```sh
|
||||
database normalize-backup [OPTIONS] [path]
|
||||
```
|
||||
|
||||
The input `path` defaults to `${LOCAL_DIR}/focus/postgresql-backup` and must be a
|
||||
backup directory compatible with `database restore`, containing `base.tar.gz`
|
||||
and optionally `pg_wal.tar.gz`. The output defaults to a normalized backup path
|
||||
and can be overridden with `--output <path>`. The output directory must not be
|
||||
the same path as the input and must not be inside the input directory.
|
||||
|
||||
Normalization extracts the physical backup into temporary PGDATA, replaces
|
||||
production PostgreSQL access/configuration with local restore-safe
|
||||
`postgresql.conf` and `pg_hba.conf` files, removes standby/recovery/runtime
|
||||
leftovers, starts the cluster locally, ensures the requested `--role` and
|
||||
`--database` exist, stops cleanly, and repacks a backup directory that can be
|
||||
used by `database restore`.
|
||||
|
||||
## Subcommands
|
||||
|
||||
- `deploy`: Execute the full deployment flow (hydrate + patch). Supports `--cleanup` to teardown after success.
|
||||
@@ -73,6 +95,7 @@ If any of these variables are missing when `pull_staging` is invoked, the tool w
|
||||
- `test`: Execute database tests located in `${DATABASE_DIR}/test/test.sql`.
|
||||
- `check`: Run a deployment validation in an isolated, temporary PostgreSQL cluster.
|
||||
- `cleanup`: Stop the local database cluster and remove the `PG_WORKING_DIR`.
|
||||
- `normalize-backup`: Rewrite a physical backup artifact for local restore use.
|
||||
- `pull_staging`: Import data from the staging environment based on the env contract.
|
||||
- `init`: Wrapper around `postgres-init` to start the cluster.
|
||||
- `migrator`: Directly invoke the migration tool with the correct environment context.
|
||||
|
||||
@@ -4,6 +4,7 @@
|
||||
: "${REMAINING_ARGS:=}"
|
||||
|
||||
: "${DEFAULT_BACKUP_PATH:=${LOCAL_DIR:-$PWD}/focus/postgresql-backup}"
|
||||
: "${DEFAULT_NORMALIZED_BACKUP_PATH:=${LOCAL_DIR:-$PWD}/focus/postgresql-backup-normalized}"
|
||||
|
||||
: "${SCRIPT_NAME:=$(basename "$0")}"
|
||||
SCRIPT_NAME=${SCRIPT_NAME%%.sh}
|
||||
@@ -186,6 +187,9 @@ ${BGREEN}Database Subcommands:${NC}
|
||||
|
||||
${BCYAN}restore${NC} ${CYAN}[PATH]$NC Restore database from backup
|
||||
|
||||
${BCYAN}normalize-backup${NC} ${CYAN}[OPTIONS] [PATH]$NC
|
||||
Normalize a raw physical backup for local restore/diff
|
||||
|
||||
${BCYAN}backup${NC} Create database backup
|
||||
Creates compressed backup at $BBLACK$DEFAULT_BACKUP_PATH$NC
|
||||
|
||||
@@ -246,6 +250,7 @@ ${BGREEN}Examples:${NC}
|
||||
$SCRIPT_NAME init Initialize PostgreSQL only
|
||||
$SCRIPT_NAME restore Restore from default backup
|
||||
$SCRIPT_NAME restore /path/to/backup Restore from specific path
|
||||
$SCRIPT_NAME normalize-backup /path/to/raw-backup
|
||||
$SCRIPT_NAME backup Create backup
|
||||
$SCRIPT_NAME log Show database logs
|
||||
$SCRIPT_NAME log list List available log files
|
||||
@@ -421,6 +426,43 @@ EOF
|
||||
)" | "$PAGER_OR_CAT"
|
||||
}
|
||||
|
||||
help_normalize_backup() {
|
||||
# shellcheck disable=SC2059
|
||||
printf "$(cat <<EOF
|
||||
${BGREEN}Usage:${NC} $SCRIPT_NAME normalize-backup [OPTIONS] [path]
|
||||
|
||||
Normalize a raw PostgreSQL physical backup for local development.
|
||||
|
||||
This command leaves the input backup untouched. It extracts $BBLACK\`base.tar.gz\`$NC
|
||||
and optional $BBLACK\`pg_wal.tar.gz\`$NC into temporary PGDATA, writes local
|
||||
$BBLACK\`postgresql.conf\`$NC and $BBLACK\`pg_hba.conf\`$NC files, removes standby,
|
||||
recovery, and runtime leftovers, starts the cluster locally, ensures a login role
|
||||
and database exist, stops cleanly, then repacks a normalized backup directory.
|
||||
|
||||
${BGREEN}Arguments:${NC}
|
||||
${BCYAN}path$NC Input backup directory (optional)
|
||||
Defaults to $BBLACK$DEFAULT_BACKUP_PATH$NC
|
||||
|
||||
${BGREEN}Options:${NC}
|
||||
$BCYAN-o$NC, $BCYAN--output$NC ${CYAN}<path>$NC Output backup directory
|
||||
Defaults to $BBLACK$DEFAULT_NORMALIZED_BACKUP_PATH$NC
|
||||
$BCYAN--role$NC ${CYAN}<name>$NC Login role to ensure (default $BBLACK\$(id -un)$NC)
|
||||
$BCYAN--database$NC ${CYAN}<name>$NC Database to ensure (default $BBLACK\${PG_DATABASE:-testdb}$NC)
|
||||
$BCYAN--admin-role$NC ${CYAN}<name>$NC Role used for local maintenance
|
||||
Defaults to $BBLACK\${URI_USER:-postgres}$NC
|
||||
$BCYAN-h$NC, $BCYAN--help$NC Show this help message
|
||||
|
||||
${BGREEN}Files Created:${NC}
|
||||
- ${BBLACK}\`base.tar.gz\`$NC: Normalized database cluster backup
|
||||
|
||||
${BGREEN}Examples:${NC}
|
||||
$SCRIPT_NAME normalize-backup /path/to/raw-backup
|
||||
$SCRIPT_NAME normalize-backup --output focus/postgresql-backup-normalized
|
||||
|
||||
EOF
|
||||
)" | "$PAGER_OR_CAT"
|
||||
}
|
||||
|
||||
help_diff() {
|
||||
# shellcheck disable=SC2059
|
||||
printf "$(cat <<EOF
|
||||
@@ -826,6 +868,195 @@ subcommand_restore() {
|
||||
restore_namespace
|
||||
}
|
||||
|
||||
___sql_literal() {
|
||||
printf "'%s'" "$(printf '%s' "$1" | sed "s/'/''/g")"
|
||||
}
|
||||
|
||||
___normalize_backup_remove_leftovers() {
|
||||
local pgdata="$1"
|
||||
rm -f \
|
||||
"$pgdata/postmaster.pid" \
|
||||
"$pgdata/postmaster.opts" \
|
||||
"$pgdata/recovery.signal" \
|
||||
"$pgdata/standby.signal" \
|
||||
"$pgdata/backup_label.old"
|
||||
rm -f "$pgdata/pg_wal/archive_status"/*.ready 2>/dev/null || :
|
||||
}
|
||||
|
||||
___normalize_backup_cleanup() {
|
||||
local tmpdir="$1"
|
||||
local pgdata="$2"
|
||||
if [ -n "$pgdata" ] && [ -d "$pgdata" ]; then
|
||||
pg_ctl -D "$pgdata" -m fast -w stop >/dev/null 2>&1 || :
|
||||
fi
|
||||
if [ -n "$tmpdir" ]; then
|
||||
rm -rf "$tmpdir"
|
||||
fi
|
||||
}
|
||||
|
||||
subcommand_normalize_backup() {
|
||||
change_namespace 'db normalize-backup'
|
||||
|
||||
NORMALIZE_INPUT_PATH=""
|
||||
NORMALIZE_OUTPUT_PATH="$DEFAULT_NORMALIZED_BACKUP_PATH"
|
||||
NORMALIZE_ROLE="$(id -un)"
|
||||
NORMALIZE_DATABASE="${PG_DATABASE:-testdb}"
|
||||
NORMALIZE_ADMIN_ROLE="${URI_USER:-postgres}"
|
||||
NORMALIZE_PORT="${PG_PORT:-5432}"
|
||||
|
||||
while [ $# -gt 0 ]; do
|
||||
case $1 in
|
||||
-h|--help)
|
||||
help_normalize_backup
|
||||
exit 0
|
||||
;;
|
||||
-o|--output)
|
||||
if [ $# -lt 2 ]; then
|
||||
log error "normalize-backup: --output requires a path"
|
||||
exit 3
|
||||
fi
|
||||
NORMALIZE_OUTPUT_PATH="$2"
|
||||
shift 2
|
||||
;;
|
||||
--role)
|
||||
if [ $# -lt 2 ]; then
|
||||
log error "normalize-backup: --role requires a name"
|
||||
exit 3
|
||||
fi
|
||||
NORMALIZE_ROLE="$2"
|
||||
shift 2
|
||||
;;
|
||||
--database)
|
||||
if [ $# -lt 2 ]; then
|
||||
log error "normalize-backup: --database requires a name"
|
||||
exit 3
|
||||
fi
|
||||
NORMALIZE_DATABASE="$2"
|
||||
shift 2
|
||||
;;
|
||||
--admin-role)
|
||||
if [ $# -lt 2 ]; then
|
||||
log error "normalize-backup: --admin-role requires a name"
|
||||
exit 3
|
||||
fi
|
||||
NORMALIZE_ADMIN_ROLE="$2"
|
||||
shift 2
|
||||
;;
|
||||
--*|-*)
|
||||
log error "normalize-backup argument $1 does not exist"
|
||||
exit 9
|
||||
;;
|
||||
*)
|
||||
if [ -n "$NORMALIZE_INPUT_PATH" ]; then
|
||||
log error "normalize-backup: unexpected argument $1"
|
||||
exit 9
|
||||
fi
|
||||
NORMALIZE_INPUT_PATH="$1"
|
||||
shift
|
||||
;;
|
||||
esac
|
||||
done
|
||||
|
||||
if [ -z "$NORMALIZE_INPUT_PATH" ]; then
|
||||
NORMALIZE_INPUT_PATH="$DEFAULT_BACKUP_PATH"
|
||||
fi
|
||||
if [ -z "$NORMALIZE_ROLE" ] || [ -z "$NORMALIZE_DATABASE" ] || [ -z "$NORMALIZE_ADMIN_ROLE" ]; then
|
||||
log error "normalize-backup: role, database, and admin role must be non-empty"
|
||||
exit 3
|
||||
fi
|
||||
if [ ! -d "$NORMALIZE_INPUT_PATH" ]; then
|
||||
log error "normalize-backup: input backup directory not found: $NORMALIZE_INPUT_PATH"
|
||||
exit 3
|
||||
fi
|
||||
if [ ! -f "$NORMALIZE_INPUT_PATH/base.tar.gz" ]; then
|
||||
log error "normalize-backup: missing $NORMALIZE_INPUT_PATH/base.tar.gz"
|
||||
exit 3
|
||||
fi
|
||||
|
||||
NORMALIZE_INPUT_ABS=$(realpath "$NORMALIZE_INPUT_PATH")
|
||||
NORMALIZE_OUTPUT_ABS=$(realpath -m "$NORMALIZE_OUTPUT_PATH")
|
||||
|
||||
if [ "$NORMALIZE_INPUT_ABS" = "$NORMALIZE_OUTPUT_ABS" ]; then
|
||||
log error "normalize-backup: input and output paths must differ"
|
||||
exit 1
|
||||
fi
|
||||
case "$NORMALIZE_OUTPUT_ABS/" in
|
||||
"$NORMALIZE_INPUT_ABS"/*)
|
||||
log error "normalize-backup: output path must not be inside input backup"
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
case "$NORMALIZE_INPUT_ABS/" in
|
||||
"$NORMALIZE_OUTPUT_ABS"/*)
|
||||
log error "normalize-backup: input backup must not be inside output path"
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
mkdir -p "$(dirname "$NORMALIZE_OUTPUT_ABS")"
|
||||
|
||||
NORMALIZE_TMPDIR=$(mktemp -d)
|
||||
NORMALIZE_PGDATA="$NORMALIZE_TMPDIR/data"
|
||||
NORMALIZE_SOCKDIR="$NORMALIZE_TMPDIR/sock"
|
||||
NORMALIZE_LOG="$NORMALIZE_TMPDIR/postgres.log"
|
||||
trap '___normalize_backup_cleanup "$NORMALIZE_TMPDIR" "$NORMALIZE_PGDATA"' EXIT INT HUP
|
||||
|
||||
mkdir -m 700 "$NORMALIZE_PGDATA"
|
||||
mkdir -p "$NORMALIZE_SOCKDIR" "$NORMALIZE_PGDATA/pg_wal"
|
||||
|
||||
log notice "extracting backup into temporary PGDATA"
|
||||
tar -xzf "$NORMALIZE_INPUT_ABS/base.tar.gz" -C "$NORMALIZE_PGDATA"
|
||||
if [ -f "$NORMALIZE_INPUT_ABS/pg_wal.tar.gz" ]; then
|
||||
tar -xzf "$NORMALIZE_INPUT_ABS/pg_wal.tar.gz" -C "$NORMALIZE_PGDATA/pg_wal"
|
||||
fi
|
||||
|
||||
___normalize_backup_remove_leftovers "$NORMALIZE_PGDATA"
|
||||
|
||||
cat > "$NORMALIZE_PGDATA/postgresql.conf" <<EOF
|
||||
listen_addresses = ''
|
||||
port = $NORMALIZE_PORT
|
||||
unix_socket_directories = '$NORMALIZE_SOCKDIR'
|
||||
logging_collector = off
|
||||
shared_preload_libraries = ''
|
||||
EOF
|
||||
cat > "$NORMALIZE_PGDATA/pg_hba.conf" <<EOF
|
||||
local all all trust
|
||||
EOF
|
||||
cat > "$NORMALIZE_PGDATA/postgresql.auto.conf" <<EOF
|
||||
# Local-dev normalized backup. Runtime ALTER SYSTEM settings intentionally cleared.
|
||||
EOF
|
||||
|
||||
log notice "starting temporary local cluster"
|
||||
with_closed_fds pg_ctl -D "$NORMALIZE_PGDATA" -o "-F" -w start > "$NORMALIZE_LOG" 2>&1 || {
|
||||
log error "normalize-backup: failed to start temporary cluster; see $NORMALIZE_LOG"
|
||||
exit 1
|
||||
}
|
||||
|
||||
NORMALIZE_ROLE_SQL=$(___sql_literal "$NORMALIZE_ROLE")
|
||||
psql -h "$NORMALIZE_SOCKDIR" -p "$NORMALIZE_PORT" -U "$NORMALIZE_ADMIN_ROLE" -d postgres \
|
||||
-v ON_ERROR_STOP=1 -c "DO \$\$ DECLARE v_role text := $NORMALIZE_ROLE_SQL; BEGIN IF NOT EXISTS (SELECT 1 FROM pg_roles WHERE rolname = v_role) THEN EXECUTE format('CREATE ROLE %I LOGIN', v_role); END IF; END \$\$;"
|
||||
|
||||
NORMALIZE_DATABASE_SQL=$(___sql_literal "$NORMALIZE_DATABASE")
|
||||
NORMALIZE_DATABASE_EXISTS=$(psql -h "$NORMALIZE_SOCKDIR" -p "$NORMALIZE_PORT" -U "$NORMALIZE_ADMIN_ROLE" -d postgres \
|
||||
-tAc "SELECT 1 FROM pg_database WHERE datname = $NORMALIZE_DATABASE_SQL" 2>/dev/null || true)
|
||||
if [ "$NORMALIZE_DATABASE_EXISTS" != "1" ]; then
|
||||
createdb -h "$NORMALIZE_SOCKDIR" -p "$NORMALIZE_PORT" -U "$NORMALIZE_ADMIN_ROLE" \
|
||||
-O "$NORMALIZE_ROLE" "$NORMALIZE_DATABASE"
|
||||
fi
|
||||
|
||||
log notice "stopping temporary local cluster"
|
||||
pg_ctl -D "$NORMALIZE_PGDATA" -m fast -w stop >> "$NORMALIZE_LOG" 2>&1
|
||||
___normalize_backup_remove_leftovers "$NORMALIZE_PGDATA"
|
||||
|
||||
log notice "writing normalized backup to $WHITE$NORMALIZE_OUTPUT_ABS$NC"
|
||||
rm -rf "$NORMALIZE_OUTPUT_ABS"
|
||||
mkdir -p "$NORMALIZE_OUTPUT_ABS"
|
||||
tar -czf "$NORMALIZE_OUTPUT_ABS/base.tar.gz" -C "$NORMALIZE_PGDATA" .
|
||||
|
||||
___normalize_backup_cleanup "$NORMALIZE_TMPDIR" ""
|
||||
trap - EXIT INT HUP
|
||||
restore_namespace
|
||||
}
|
||||
|
||||
subcommand_log() {
|
||||
change_namespace 'db log'
|
||||
: "${PG_WORKING_DIR:="$LOCAL_DIR/focus/postgresql"}"
|
||||
@@ -1590,6 +1821,14 @@ if ! [ "${AS_LIBRARY+x}" ]; then
|
||||
DB_URL=$2
|
||||
shift 2
|
||||
;;
|
||||
normalize-backup)
|
||||
if [ "${SUBCOMMAND+x}" ]; then
|
||||
REMAINING_ARGS="$REMAINING_ARGS $(quote "$1")"
|
||||
else
|
||||
SUBCOMMAND="normalize_backup"
|
||||
fi
|
||||
shift
|
||||
;;
|
||||
deploy|replay|restore|patch|hydrate|backup|log|migrator|diff|pull_staging|test|check|cleanup|init)
|
||||
if [ "${SUBCOMMAND+x}" ]; then
|
||||
REMAINING_ARGS="$REMAINING_ARGS $(quote "$1")"
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
# shellcheck shell=dash
|
||||
|
||||
HECTIC_NAMESPACE=test-db-tool-help
|
||||
export HECTIC_NAMESPACE=test-db-tool-help
|
||||
|
||||
log notice "test case: database --help exits 0"
|
||||
if ! database --help > /tmp/help-out.txt 2>&1; then
|
||||
@@ -8,7 +8,7 @@ if ! database --help > /tmp/help-out.txt 2>&1; then
|
||||
exit 1
|
||||
fi
|
||||
|
||||
for tok in deploy pull_staging cleanup check log init migrator; do
|
||||
for tok in deploy pull_staging cleanup check log init migrator normalize-backup; do
|
||||
if ! grep -qF "$tok" /tmp/help-out.txt; then
|
||||
log error "test failed: --help output missing token: $tok"
|
||||
cat /tmp/help-out.txt >&2
|
||||
|
||||
@@ -0,0 +1,78 @@
|
||||
# shellcheck shell=dash
|
||||
|
||||
export HECTIC_NAMESPACE=test-db-tool-normalize-backup
|
||||
|
||||
tmp_root=$(mktemp -d)
|
||||
restore_root=
|
||||
trap 'PG_WORKING_DIR=$restore_root postgres-cleanup >/dev/null 2>&1 || true; pg_harness_stop; rm -rf "$tmp_root"' EXIT INT TERM
|
||||
|
||||
export LOCAL_DIR="$tmp_root/local"
|
||||
export PG_DATABASE=postgres
|
||||
mkdir -p "$LOCAL_DIR"
|
||||
|
||||
log notice "test case: normalize-backup rejects output inside input backup"
|
||||
pg_harness_start
|
||||
|
||||
current_user=$(id -un)
|
||||
psql -v ON_ERROR_STOP=1 <<SQL
|
||||
DO \$\$
|
||||
BEGIN
|
||||
IF NOT EXISTS (SELECT 1 FROM pg_roles WHERE rolname = '$current_user') THEN
|
||||
CREATE ROLE "$current_user" LOGIN SUPERUSER;
|
||||
END IF;
|
||||
END
|
||||
\$\$;
|
||||
CREATE TABLE public.normalize_backup_probe (id integer PRIMARY KEY, label text NOT NULL);
|
||||
INSERT INTO public.normalize_backup_probe VALUES (1, 'normalized artifact restored');
|
||||
SQL
|
||||
|
||||
input_backup="$tmp_root/input-backup"
|
||||
invalid_output="$input_backup/nested-output"
|
||||
output_backup="$tmp_root/output-backup"
|
||||
restore_root="$tmp_root/restore-pg"
|
||||
|
||||
mkdir -p "$input_backup"
|
||||
pg_ctl stop -D "$PGDATA" -m fast -w >/dev/null
|
||||
tar -czf "$input_backup/base.tar.gz" -C "$PGDATA" .
|
||||
unset PGDATA PGHOST PGPORT PGUSER PGDATABASE
|
||||
|
||||
set +e
|
||||
database normalize-backup --output "$invalid_output" --role "$current_user" --database postgres --admin-role postgres "$input_backup" >/tmp/normalize-invalid-out.txt 2>/tmp/normalize-invalid-err.txt
|
||||
code=$?
|
||||
set -e
|
||||
|
||||
if [ "$code" = 0 ]; then
|
||||
log error "test failed: normalize-backup allowed output inside input"
|
||||
cat /tmp/normalize-invalid-out.txt >&2
|
||||
cat /tmp/normalize-invalid-err.txt >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
log notice "test case: normalize-backup produces restorable local artifact"
|
||||
database normalize-backup --output "$output_backup" --role "$current_user" --database postgres --admin-role postgres "$input_backup"
|
||||
|
||||
if ! [ -f "$output_backup/base.tar.gz" ]; then
|
||||
log error "test failed: normalized backup missing base.tar.gz"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
tar -tzf "$output_backup/base.tar.gz" > "$tmp_root/normalized-files.txt"
|
||||
if ! grep -Eq '^(\./)?postgresql\.conf$' "$tmp_root/normalized-files.txt"; then
|
||||
log error "test failed: normalized base archive missing postgresql.conf"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
pg_harness_stop
|
||||
|
||||
mkdir -p "$restore_root"
|
||||
PG_WORKING_DIR="$restore_root" database restore "$output_backup"
|
||||
|
||||
if ! psql -h "$restore_root/sock" -p 5432 -U "$current_user" -d postgres -tAc "SELECT label FROM public.normalize_backup_probe WHERE id = 1" | grep -q '^normalized artifact restored$'; then
|
||||
log error "test failed: normalized backup did not restore usable local data"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
PG_WORKING_DIR="$restore_root" postgres-cleanup
|
||||
restore_root=
|
||||
|
||||
log notice "test passed"
|
||||
Reference in New Issue
Block a user