diff --git a/legacy/helper/posix-shell/change_namespace.sh b/legacy/helper/posix-shell/change_namespace.sh new file mode 100644 index 0000000..e6f805a --- /dev/null +++ b/legacy/helper/posix-shell/change_namespace.sh @@ -0,0 +1,41 @@ +: "${OLD_NAMESPACE:=}" + +nl=$(printf '\nx') +nl=${nl%x} + +___pop_namespace() { + v=${OLD_NAMESPACE%%"$nl"*} + + case $OLD_NAMESPACE in + *"$nl"*) + OLD_NAMESPACE=${OLD_NAMESPACE#*"$nl"} + ;; + *) + OLD_NAMESPACE= + ;; + esac + + printf '%s\n' "$v" +} + +___peek_namespace() { + printf '%s\n' "${OLD_NAMESPACE%%"$nl"*}" +} + +___push_namespace() { + if [ -n "$OLD_NAMESPACE" ]; then + OLD_NAMESPACE=$1"$nl$OLD_NAMESPACE" + else + OLD_NAMESPACE=$1 + fi +} + +change_namespace() { + ___push_namespace "$HECTIC_NAMESPACE" + export HECTIC_NAMESPACE="$1" +} + +restore_namespace() { + HECTIC_NAMESPACE=$(___pop_namespace) + export HECTIC_NAMESPACE +} diff --git a/legacy/helper/posix-shell/default.nix b/legacy/helper/posix-shell/default.nix index bbf9a31..5a7ed47 100644 --- a/legacy/helper/posix-shell/default.nix +++ b/legacy/helper/posix-shell/default.nix @@ -12,4 +12,13 @@ in { colors = hectic.writeDash "colors.sh" '' ${builtins.readFile ./colors.sh} ''; + change_namespace = hectic.writeDash "change_namespace.sh" '' + ${builtins.readFile ./change_namespace.sh} + ''; + quote = hectic.writeDash "quote.sh" '' + ${builtins.readFile ./quote.sh} + ''; + pager_or_cat = hectic.writeDash "pager_or_cat.sh" '' + ${builtins.readFile ./pager_or_cat.sh} + ''; } diff --git a/legacy/helper/posix-shell/pager_or_cat.sh b/legacy/helper/posix-shell/pager_or_cat.sh new file mode 100644 index 0000000..219c660 --- /dev/null +++ b/legacy/helper/posix-shell/pager_or_cat.sh @@ -0,0 +1,8 @@ +pager_or_cat_init() { + # Pipe to pager only if stdout is a terminal, otherwise output directly + if [ -t 1 ]; then + PAGER_OR_CAT="${PAGER:-less}" + else + PAGER_OR_CAT=cat + fi +} diff --git a/legacy/helper/posix-shell/quote.sh b/legacy/helper/posix-shell/quote.sh new file mode 100644 index 0000000..5d7a59b --- /dev/null +++ b/legacy/helper/posix-shell/quote.sh @@ -0,0 +1 @@ +quote() { printf "'%s'" "$(printf %s "$1" | sed "s/'/'\\\\''/g")"; } diff --git a/package/db-tool/README.md b/package/db-tool/README.md new file mode 100644 index 0000000..e265599 --- /dev/null +++ b/package/db-tool/README.md @@ -0,0 +1,99 @@ +# db-tool + +PostgreSQL development database management tool. Drop-in replacement for per-project database.sh / postgres-init.sh / postgres-cleanup.sh scripts. Provides database, postgres-init, and postgres-cleanup binaries. + +## Provided Binaries + +| Binary | Description | +| --- | --- | +| `database` | Main script for managing migrations, deployments, and logs. | +| `postgres-init` | Ephemeral PostgreSQL cluster initialization and startup. | +| `postgres-cleanup` | Graceful shutdown and cleanup of the PostgreSQL cluster. | + +## Required Environment Variables + +These variables must be set for `db-tool` to function. + +| Variable | Description | +| --- | --- | +| `LOCAL_DIR` | Absolute path to the project root directory. | +| `DB_URL` | Full PostgreSQL connection string (e.g., `postgresql://user@localhost/dbname?host=$PG_WORKING_DIR`). | +| `PG_WORKING_DIR` | Directory where the PostgreSQL cluster data and sockets are stored. | + +## Optional Environment Variables + +| Variable | Default Value | Description | +| --- | --- | --- | +| `DATABASE_DIR` | `${LOCAL_DIR}/db` | Root directory for database-related files. | +| `MIGRATION_DIR` | `${DATABASE_DIR}/migration` | Directory containing SQL migration files. | +| `DATABASE_SOURCE` | `${DATABASE_DIR}/src` | Directory containing source SQL files for hydration. | +| `PG_URL_VAR` | `PGURL` | The name of the environment variable where the computed PG URL will be exported. | +| `PG_LOG_PATH` | (unset) | Path to redirect PostgreSQL server logs. | +| `PATCH_LOG` | (stdout) | Path to log the output of database patches. | +| `HYDRATE_LOG` | (stdout) | Path to log the output of database hydration. | + +## pull_staging Contract + +The `pull_staging` subcommand allows importing data from a remote staging environment into the local `test-data.sql` file. This functionality requires four specific environment variables to be defined: + +1. `STAGING_SSH_HOST`: The SSH destination for the staging server. +2. `STAGING_DB_URL`: The PostgreSQL connection string for the remote staging database. +3. `STAGING_DUMP_TABLES`: A space-separated list of tables to include in the data dump. +4. `STAGING_DUMP_FLAGS`: Additional flags to pass to `pg_dump` (e.g., `--column-inserts`). + +If any of these variables are missing when `pull_staging` is invoked, the tool will exit with code 3 and print the name of the missing variable to stderr. + +## Subcommands + +- `deploy`: Execute the full deployment flow (hydrate + patch). Supports `--cleanup` to teardown after success. +- `log`: Inspect database logs. Supports `list` and index-based selection. +- `test`: Execute database tests located in `${DATABASE_DIR}/test/test.sql`. +- `check`: Run a deployment validation in an isolated, temporary PostgreSQL cluster. +- `cleanup`: Stop the local database cluster and remove the `PG_WORKING_DIR`. +- `pull_staging`: Import data from the staging environment based on the env contract. +- `init`: Wrapper around `postgres-init` to start the cluster. +- `migrator`: Directly invoke the migration tool with the correct environment context. + +## shellHook Example + +To use `db-tool` in a Nix development shell, add the following to your `flake.nix` or `shell.nix`: + +```nix +{ + # ... + devShells.default = pkgs.mkShell { + packages = [ + pkgs.hectic.db-tool + pkgs.hectic.postgres-init + pkgs.hectic.postgres-cleanup + ]; + + shellHook = '' + export LOCAL_DIR="$PWD" + export DATABASE_DIR="$LOCAL_DIR/db" + export MIGRATION_DIR="$DATABASE_DIR/migration" + export DATABASE_SOURCE="$DATABASE_DIR/src" + export PG_WORKING_DIR="$LOCAL_DIR/focus/postgresql" + export DB_URL="postgresql://user@localhost/dbname?host=$PG_WORKING_DIR&port=5432" + + # for other non-db scripts (deploy.sh, task.sh, etc.): + export HECTIC_LIB="${pkgs.hectic.helpers.posix-shell.log}" + + # Initialize and start the ephemeral database cluster + . ${pkgs.hectic.postgres-init}/bin/postgres-init + ''; + }; +} +``` + +## Exit Codes + +| Code | Meaning | +| --- | --- | +| 1 | Generic error. | +| 2 | Ambiguous arguments or state. | +| 3 | Missing required argument or environment variable. | +| 5 | Provided table does not exist. | +| 9 | Argument or command not found. | +| 13 | Program bug or unexpected system state. | +| 127 | Command not found (missing dependency). | diff --git a/package/db-tool/database.sh b/package/db-tool/database.sh new file mode 100644 index 0000000..0fd881e --- /dev/null +++ b/package/db-tool/database.sh @@ -0,0 +1,1539 @@ +# shellcheck shell=dash +# shellcheck disable=SC3043 + +: "${REMAINING_ARGS:=}" + +: "${DEFAULT_BACKUP_PATH:=${LOCAL_DIR:-$PWD}/focus/postgresql-backup/}" + +: "${SCRIPT_NAME:=$(basename "$0")}" +SCRIPT_NAME=${SCRIPT_NAME%%.sh} + +: "${PATCH_SQL:=${PROFILE_DIR:-${LOCAL_DIR:-$PWD}}/test-data.sql}" + +if [ "${PG_LOG_PATH+x}" ]; then + : "${PATCH_LOG:=${PG_LOG_PATH}/patch.stdout.log}" + : "${HYDRATE_LOG:=${PG_LOG_PATH}/hydate.stdout.log}" +fi + +: "${DATABASE_DIR:=${LOCAL_DIR:-$PWD}/db}" +: "${MIGRATION_DIR:=$DATABASE_DIR/migration}" +: "${DATABASE_SOURCE:=$DATABASE_DIR/src}" + +: "${HYDRATE_ENTRYPOINT:=entrypoint.sql}" + +HYDRATE_ENTRYPOINT="${DATABASE_SOURCE}/${HYDRATE_ENTRYPOINT}" + +: "${ENVIRONMENT:=}" + +pager_or_cat_init + +: "${DB_URL:=${PGURL:-}}" + +form_psql_args() { + local psql_args="$PGURL -v ON_ERROR_STOP=1" + #for var in ${VARIABLE_LIST:-}; do + # psql_args="$psql_args -v $var" + #done + printf '%s' "$psql_args" +} + +psql_logged() { + local log_file="$1" + shift + if [ -n "$log_file" ]; then + psql "$@" > "$log_file" + else + psql "$@" + fi +} + +db_exec() { + local sql="$1" + # shellcheck disable=SC2046 + printf '%s' "$sql" | psql $(form_psql_args) +} + +todo() { + log panic "TODO" + exit 1 +} + +log_pager() { + # shellcheck disable=SC2068 + nvim -R \ + -u NONE \ + -c 'nnoremap q :qa!' \ + -c 'runtime! plugin/*.vim' \ + -c 'set conceallevel=3' \ + $@ \ + - +} + +help_log() { + # shellcheck disable=SC2059 + printf "$(cat <|] + +View PostgreSQL logs for current PG_WORKING_DIR. + +${BGREEN}Modes:${NC} + ${BCYAN}log$NC Interactive pager when attached to TTY, + non-interactive concat when piped/redirected + ${BCYAN}log list$NC List available log files with numeric indexes + ${BCYAN}log $NC Open/print one file by numeric index from list + ${BCYAN}log $NC Open/print one file by exact name or unique substring + +${BGREEN}Examples:${NC} + $SCRIPT_NAME log + $SCRIPT_NAME log list + $SCRIPT_NAME log 1 + $SCRIPT_NAME log postgresql-2026-04-21 + +EOF +)" | "$PAGER_OR_CAT" +} + +db_log_list_files() { + log_dir="$1" + for log_file in "$log_dir"/*; do + [ -f "$log_file" ] || continue + printf '%s\n' "$log_file" + done +} + +db_log_print_list() { + log_dir="$1" + idx=1 + found=0 + for log_file in "$log_dir"/*; do + [ -f "$log_file" ] || continue + found=1 + printf '%2s %s\n' "$idx" "$(basename "$log_file")" + idx=$((idx + 1)) + done + if [ "$found" -eq 0 ]; then + log warn "no log files in $log_dir" + fi +} + +db_log_resolve_one() { + log_dir="$1" + selector="$2" + + if [ -f "$log_dir/$selector" ]; then + printf '%s\n' "$log_dir/$selector" + return 0 + fi + + if [ -n "$selector" ] && [ "$selector" -eq "$selector" ] 2>/dev/null; then + idx=1 + for log_file in "$log_dir"/*; do + [ -f "$log_file" ] || continue + if [ "$idx" -eq "$selector" ]; then + printf '%s\n' "$log_file" + return 0 + fi + idx=$((idx + 1)) + done + return 1 + fi + + matches= + match_count=0 + for log_file in "$log_dir"/*; do + [ -f "$log_file" ] || continue + case "$(basename "$log_file")" in + *"$selector"*) + match_count=$((match_count + 1)) + matches="$matches$(printf '%s\n' "$log_file")" + ;; + esac + done + + if [ "$match_count" -eq 1 ]; then + printf '%s' "$matches" + return 0 + fi + + return 1 +} + +help() { + # shellcheck disable=SC2059 + printf "$(cat < [OPTIONS] + +PostgreSQL development database management + +${BGREEN}Global Options:${NC} + $BCYAN-h$NC, $BCYAN--help$NC Show this help message + $BCYAN-i$NC, $BCYAN--inherits$NC Parent table for hectic.migration + Can be specified multiple times + $BCYAN-u$NC, $BCYAN--url$NC PostgreSQL connection string (overrides PGURL) + +${BGREEN}Database Subcommands:${NC} + ${BCYAN}deploy ${CYAN}[OPTIONS]$NC Initialize and deploy PostgreSQL database + Default action with full setup including hydrate and patch + Options: + $BCYAN--no-patch$NC Skip applying test-data.sql + $BCYAN--no-hydrate$NC Skip building from source files + + ${BCYAN}init${NC} ${CYAN}[OPTIONS]$NC Initialize PostgreSQL cluster without hydrate or patch + + ${BCYAN}restore${NC} ${CYAN}[PATH]$NC Restore database from backup + + ${BCYAN}backup${NC} Create database backup + Creates compressed backup at $BBLACK$DEFAULT_BACKUP_PATH$NC + + ${BCYAN}patch${NC} ${CYAN}[OPTIONS]$NC Patch file: $BBLACK$PATCH_SQL$NC + Apply patch to current database + Uses patch file if exists + Creates empty patch file if not found + Options: + $BCYAN--edit$NC, $BCYAN-e$NC Edit test-data.sql in \$EDITOR before applying + + ${BCYAN}hydrate${NC} ${CYAN}[OPTIONS]$NC Build database from source files + Runs $BBLACK$HYDRATE_ENTRYPOINT$NC + + ${BCYAN}migrator${NC} ${CYAN}[OPTIONS]$NC Run database migrations + Uses for production or pre production + + ${BCYAN}log${NC} ${CYAN}[ARG]$NC Show PostgreSQL logs + ARG: ${BBLACK}list$NC, numeric index, or filename/substring + No ARG: interactive on TTY, concat output when piped + + ${BCYAN}replay${NC} ${CYAN}[OPTIONS]$NC Restore database from point to point using SQL + Useful for development after minor schema changes + ${BRED}WIP$NC + + ${BCYAN}pull_staging${NC} Pull data from staging DB into test-data.sql + Uses STAGING_* environment variables for connection and dump selection + + ${BCYAN}format${NC} Format database source files + ${BRED}WIP$NC + + ${BCYAN}test${NC} Run SQL test suite (BEGIN/ROLLBACK, no side effects) + Executes ${BBLACK}${DATABASE_DIR}/test/test.sql$NC + + ${BCYAN}diff${NC} ${CYAN}[OPTIONS] [PATH]$NC Compare backup+migrations vs current sources + Shows schema differences between production and dev + Creates isolated instances for safe comparison + Subcommands: + ${BCYAN}log$NC Show diff operation logs + + ${BCYAN}check${NC} ${CYAN}[OPTIONS]$NC Full deploy + cleanup in an isolated temporary cluster + Does not affect your running development database + Options: + $BCYAN--no-patch$NC Skip applying test-data.sql + $BCYAN--no-hydrate$NC Skip building from source files + $BCYAN-m$NC Mock external API calls + + ${BCYAN}cleanup${NC} Stop the running PostgreSQL cluster and remove its working directory + +${BGREEN}Environment Variables:${NC} + ${BBLACK}PGURL$NC PostgreSQL connection string (auto-detected) + ${BBLACK}PROFILE_DIR$NC Profile directory (auto-detected) + ${BBLACK}DATABASE_SOURCE$NC Database source files directory (default $BBLACK$DATABASE_SOURCE$NC) + +${BGREEN}Examples:${NC} + ${BBLACK}# Basic operations$NC + $SCRIPT_NAME deploy Deploy database with full setup + $SCRIPT_NAME deploy --no-patch Deploy without test data + $SCRIPT_NAME init Initialize PostgreSQL only + $SCRIPT_NAME restore Restore from default backup + $SCRIPT_NAME restore /path/to/backup Restore from specific path + $SCRIPT_NAME backup Create backup + $SCRIPT_NAME log Show database logs + $SCRIPT_NAME log list List available log files + $SCRIPT_NAME log 1 Show one log file by index + $SCRIPT_NAME log postgresql-2026-04-21 Show one log file by name pattern + $SCRIPT_NAME patch Apply test data + $SCRIPT_NAME patch --edit Edit test data and apply + + ${BBLACK}# Schema comparison$NC + $SCRIPT_NAME diff Compare backup vs sources + $SCRIPT_NAME diff -o diff.txt Save comparison to file + $SCRIPT_NAME diff --with-data Include data in comparison + + ${BBLACK}# Isolated validation$NC + $SCRIPT_NAME check Full deploy + cleanup in temp cluster + $SCRIPT_NAME check -P Check without test data + $SCRIPT_NAME check -H Check without hydrating schema + +EOF +)" | "$PAGER_OR_CAT" +} + +help_patch() { + # shellcheck disable=SC2059 + printf "$(cat <${NC} Comma-separated list of tables to diff + Example: --tables users,orders,products + $BCYAN-m${NC}, $BCYAN--mock${NC} Mock external API calls when hydrating DB2 + Replaces HTTP-calling functions with stubs/test data + $BCYAN-h${NC}, $BCYAN--help${NC} Show this help message + +${BGREEN}Subcommands:${NC} + ${BCYAN}log${NC} Show PostgreSQL logs from diff operation + Displays logs from both DB1 and DB2 instances + +${BGREEN}Process:${NC} + 1. Create two isolated PostgreSQL instances (DB1 and DB2) + 2. Restore DB1 from backup and apply all migrations + 3. Hydrate DB2 from current source files + 4. Dump both databases to SQL files + 5. Compare dumps and present diff + 6. Clean up temporary instances + +${BGREEN}Environment Variables:${NC} + ${BBLACK}DATABASE_SOURCE$NC Path to database source files + +${BGREEN}Examples:${NC} + $SCRIPT_NAME diff Compare using default backup + $SCRIPT_NAME diff /path/to/backup Compare using specific backup + $SCRIPT_NAME diff --tables users,orders Compare specific tables + $SCRIPT_NAME diff -m Compare with external APIs mocked + $SCRIPT_NAME diff log Show logs from diff operation + +${BGREEN}Notes:${NC} + - Does not affect your development database + - Requires backup created with $BBLACK\`backup\`$NC subcommand + +EOF +)" | "$PAGER_OR_CAT" +} + +help_init() { + # shellcheck disable=SC2059 + printf "$(cat <&2 + exit 3 + fi + if [ -z "${STAGING_DB_URL-}" ]; then + printf 'STAGING_DB_URL is not set\n' >&2 + exit 3 + fi + if [ -z "${STAGING_DUMP_TABLES-}" ]; then + printf 'STAGING_DUMP_TABLES is not set\n' >&2 + exit 3 + fi + if [ -z "${STAGING_DUMP_FLAGS-}" ]; then + printf 'STAGING_DUMP_FLAGS is not set\n' >&2 + exit 3 + fi + + while [ $# -gt 0 ]; do + case $1 in + -h|--help) + help_pull_staging + exit 0 + ;; + --*|-*) + log error "pull_staging argument $1 does not exist" + exit 9 + ;; + *) + log error "pull_staging: unexpected argument $1" + exit 9 + ;; + esac + done + + PG_DUMP_TABLE_ARGS='' + for tbl in $STAGING_DUMP_TABLES; do + PG_DUMP_TABLE_ARGS="$PG_DUMP_TABLE_ARGS -t $(quote "$tbl")" + done + + log notice "pulling data from ${WHITE}${STAGING_SSH_HOST}${NC}" + + remote_dump_command="pg_dump $(quote "$STAGING_DB_URL") $STAGING_DUMP_FLAGS$PG_DUMP_TABLE_ARGS 2>/dev/null" + # shellcheck disable=SC2029 + STAGING_DUMP=$(ssh "$STAGING_SSH_HOST" "$remote_dump_command") || { + log error "failed to dump staging database via SSH" + exit 1 + } + + if [ -z "$STAGING_DUMP" ]; then + log error "staging dump returned empty output" + exit 1 + fi + + SEQ_RESETS='' + for tbl in $STAGING_DUMP_TABLES; do + escaped_tbl=$(printf '%s' "$tbl" | sed "s/'/''/g") + SEQ_RESETS="$SEQ_RESETS +SELECT setval(seq_name, COALESCE((SELECT MAX(\"id\") FROM $tbl), 1)) +FROM (SELECT pg_get_serial_sequence('$escaped_tbl', 'id') AS seq_name) AS seq +WHERE seq_name IS NOT NULL;" + done + + mkdir -p "$(dirname "$PATCH_SQL")" + touch "$PATCH_SQL" + cat > "$PATCH_SQL" < "$LOG_HELP" + + # shellcheck disable=SC2046 + set -- $(db_log_list_files "$LOG_DIR") + if [ "$#" -eq 0 ]; then + log warn "no log files in $LOG_DIR" + restore_namespace + return 0 + fi + + if [ "$#" -gt 1 ]; then + log_pager -p "$LOG_HELP" "$@" + else + log_pager "$1" + fi + else + found=0 + for log_file in "$LOG_DIR"/*; do + [ -f "$log_file" ] || continue + found=1 + cat "$log_file" + done + if [ "$found" -eq 0 ]; then + log warn "no log files in $LOG_DIR" + fi + fi + + restore_namespace +} + +subcommand_replay() { + change_namespace 'db replay' + todo +} + + + +# shellcheck disable=SC2120 +subcommand_hydrate() { + change_namespace 'db hydrate' + + while [ $# -gt 0 ]; do + case $1 in + -h|--help) + help_hydrate + exit 0 + ;; + -H|--no-hook) + HYDRATE_NO_HOOK=1 + shift + ;; + -m|--mock) + HYDRATE_USE_MOCK=1 + shift + ;; + --*|-*) + log error "hydrate argument $1 does not exist" + exit 9 + ;; + *) + log error "hydrate subcommand $1 does not exist" + exit 9 + ;; + esac + done + + if [ ! "${HYDRATE_NO_HOOK+x}" ]; then + log info "hectic secrets hook" + # shellcheck disable=SC2059 + printf "${BBLACK}" + sh "${LOCAL_DIR}/lib/hook/postgres-secrets.sh" "$PGURL" "$ENVIRONMENT" + + # shellcheck disable=SC2059 + printf "${NC}" + else + log info "skipping hectic secrets hook" + fi + + local mock_arg="" + if [ "${HYDRATE_USE_MOCK+x}" ]; then + log info "mock mode enabled — external API calls will be stubbed" + mock_arg="-v USE_MOCK=1" + fi + + log notice "hydrate database sources" + # shellcheck disable=SC2059 + printf "${BBLACK}" + + # shellcheck disable=SC2046 + # shellcheck disable=SC2086 + if psql_logged "${HYDRATE_LOG:-}" $(form_psql_args) $mock_arg \ + -f "$HYDRATE_ENTRYPOINT"; then + log notice "hydrating succes" + else + log error "hydrate error, check $WHITE${HYDRATE_LOG:-stdout}$NC for more" + exit 1 + fi + + restore_namespace +} + +subcommand_migrator() { + change_namespace 'db migrator' + db_url="${DB_URL:-$PGURL}" + env MIGRATION_DIR="$MIGRATION_DIR" DB_URL="$db_url" migrator "$@" + migrator_exit_code="$?" + restore_namespace + + return "$migrator_exit_code" +} + +# ___parse_deploy_flags -- shared option parser for deploy/check +# Sets: DEPLOY_NO_PATCH, DEPLOY_NO_HYDRATE, HYDRATE_USE_MOCK, DEPLOY_REUSE +# Returns remaining unparsed args via REMAINING_ARGS (caller resets first) +# Caller must supply a CALLER name for error messages (first positional arg). +___parse_deploy_flags() { + _caller="${1:?___parse_deploy_flags: caller name required}"; shift + while [ $# -gt 0 ]; do + case $1 in + -P|--no-patch) + DEPLOY_NO_PATCH=1 + shift + ;; + -H|--no-hydrate) + DEPLOY_NO_HYDRATE=1 + shift + ;; + -m|--mock) + HYDRATE_USE_MOCK=1 + shift + ;; + --reuse) + DEPLOY_NO_PATCH=1 + DEPLOY_NO_HYDRATE=1 + DEPLOY_REUSE=1 + shift + ;; + *) + log error "$_caller: unknown argument $1" + exit 9 + ;; + esac + done +} + +# ___run_deploy_flow -- shared hydrate+patch execution for deploy/check +___run_deploy_flow() { + if [ ! "${DEPLOY_NO_HYDRATE+x}" ]; then + subcommand_hydrate + fi + if [ ! "${DEPLOY_NO_PATCH+x}" ]; then + subcommand_patch + fi +} + +subcommand_init() { + : "${PG_WORKING_DIR:=${LOCAL_DIR}/focus/postgresql}" + # - to set postgresql server + # data & sock directory + + change_namespace 'db init' + unset DEPLOY_REUSE + + while [ $# -gt 0 ]; do + case $1 in + -h|--help) + help_init + exit 0 + ;; + --reuse) + DEPLOY_REUSE=1 + shift + ;; + --*|-*) + log error "init argument $1 does not exist" + exit 9 + ;; + *) + log error "init: unexpected argument $1" + exit 9 + ;; + esac + done + + { + [ "${DEPLOY_REUSE+x}" ] && export PG_REUSE + "$SHELL" "${LOCAL_DIR}/devshell/postgres-init.sh" + } + + restore_namespace +} + +subcommand_deploy() { + + : "${PG_WORKING_DIR:="$LOCAL_DIR/focus/postgresql"}" + # - to set postgresql server + # data & sock directory + + change_namespace 'db deploy' + + unset DEPLOY_NO_PATCH DEPLOY_NO_HYDRATE DEPLOY_REUSE DEPLOY_CLEANUP + _deploy_extra= + + while [ $# -gt 0 ]; do + case $1 in + -h|--help) + help_deploy + exit 0 + ;; + -c|--cleanup) + DEPLOY_CLEANUP=1 + shift + ;; + *) + _deploy_extra="$_deploy_extra $(quote "$1")" + shift + ;; + esac + done + + [ "$_deploy_extra" = '' ] || eval "___parse_deploy_flags 'deploy' $_deploy_extra" + + { + [ "${DEPLOY_REUSE+x}" ] && export PG_REUSE + "$SHELL" "${LOCAL_DIR}/devshell/postgres-init.sh" + } + + ___run_deploy_flow + + if [ "${DEPLOY_CLEANUP+x}" ]; then + log info "cleanup: stopping postgresql" + "$SHELL" "${LOCAL_DIR}/devshell/postgres-cleanup.sh" + fi + + restore_namespace +} + +# shellcheck disable=SC2120 +subcommand_patch() { + change_namespace 'db patch' + + while [ $# -gt 0 ]; do + case $1 in + -e|--edit) + PATCH_EDIT=1 + shift + ;; + -h|--help) + help_patch + exit 0 + ;; + --*|-*) + log error "patch argument $1 does not exists" + exit 9 + ;; + *) + log error "patch subcommand $1 does not exists" + exit 9 + ;; + esac + done + + mkdir -p "$(dirname "$PATCH_SQL")" + touch "$PATCH_SQL" + + if [ "${PATCH_EDIT+x}" ]; then + log notice "opening $WHITE$PATCH_SQL$NC in \$EDITOR" + "${EDITOR:-vi}" "$PATCH_SQL" + exit 0 + fi + + log notice "retoring $WHITE$PATCH_SQL$NC in database" + + # shellcheck disable=SC2059 + printf "${BBLACK}" + + # shellcheck disable=SC2046 + if psql_logged "${PATCH_LOG:-}" $(form_psql_args) -f "$PATCH_SQL"; then + log notice "restoring succes" + else + log error "error, check $WHITE${PATCH_LOG:-stdout}$NC for more" + exit 1 + fi + + restore_namespace +} + +___diff_dump_schema() { + local socket_dir="$1" + local port="$2" + local output_file="$3" + local tables="${4:-}" + + log info "dumping schema to $WHITE$output_file$NC" + + if [ -n "$tables" ]; then + # Dump specific tables with data + local table_args="" + old_IFS="$IFS" + IFS=',' + for table in $tables; do + table_args="$table_args -t $table" + done + IFS="$old_IFS" + + # shellcheck disable=SC2086 + pg_dump -h "$socket_dir" -p "$port" testdb \ + --schema-only --no-owner --no-privileges \ + $table_args > "$output_file" 2>/dev/null + + # shellcheck disable=SC2086 + pg_dump -h "$socket_dir" -p "$port" testdb \ + --data-only --no-owner --no-privileges \ + $table_args >> "$output_file" 2>/dev/null + else + # Schema only + pg_dump -h "$socket_dir" -p "$port" testdb \ + --schema-only --no-owner --no-privileges \ + > "$output_file" 2>/dev/null + fi +} + +help_check() { + # shellcheck disable=SC2059 + printf "$(cat </dev/null 2>&1 || true; pg_ctl -D "$DIFF_TMPDIR/pgdata2/data" stop -m fast >/dev/null 2>&1 || true;' EXIT INT HUP + + mkdir -p "$DIFF_PGDATA1" "$DIFF_PGDATA2" + log info "DB1: $WHITE$DIFF_PGDATA1$NC" + log info "DB2: $WHITE$DIFF_PGDATA2$NC" + + log notice "comparing backup vs sources" + log info "backup: $WHITE$DIFF_BACKUP_PATH$NC" + log info "sources: $WHITE$DATABASE_SOURCE$NC" + + log notice "provisioning ${WHITE}DB1$NC (backup + migrations)" + + PG_WORKING_DIR="$DIFF_PGDATA1" PG_LOG_PATH="$DIFF_PGDATA1" subcommand_restore + + log info "applying migrations to ${WHITE}DB1$NC" + subcommand_migrator migrate up all \ + --db-url \ + "$DIFF_PGURL1" \ + || { + log warn "migrations failed or none to apply" + } + + log notice "provisioning ${WHITE}DB2$NC (current sources)" + + log info "initializing ${WHITE}DB2$NC with postgres-init.sh" + PG_WORKING_DIR="$DIFF_PGDATA2" \ + PG_DATABASE="testdb" \ + PG_DISABLE_LOGGING=1 \ + "$SHELL" "${LOCAL_DIR}/devshell/postgres-init.sh" || { + log error "failed to initialize ${WHITE}DB2$NC" + exit 1 + } + + log info "hydrating ${WHITE}DB2$NC from sources" + PGURL="$DIFF_PGURL2" \ + subcommand_hydrate || { + log error "hydration failed" + exit 1 + } + + subcommand_migrator init \ + --db-url \ + "$DIFF_PGURL2" \ + || { + log error "init migration failed" + exit 1 + } + + log notice "dumping schemas" + DIFF_DUMP1="$DIFF_TMPDIR/target.sql" + DIFF_DUMP2="$DIFF_TMPDIR/source.sql" + + ___diff_dump_schema "$DIFF_PGDATA1/sock" "5432" "$DIFF_DUMP1" "$DIFF_TABLES" + ___diff_dump_schema "$DIFF_PGDATA2/sock" "5432" "$DIFF_DUMP2" "$DIFF_TABLES" + + # Optional: filter out cron tables + if [ "$DIFF_NO_CRON" = "1" ]; then + log info "filtering cron tables" + grep -v "cron\." "$DIFF_DUMP1" > "$DIFF_DUMP1.filtered" 2>/dev/null || true + grep -v "cron\." "$DIFF_DUMP2" > "$DIFF_DUMP2.filtered" 2>/dev/null || true + mv "$DIFF_DUMP1.filtered" "$DIFF_DUMP1" + mv "$DIFF_DUMP2.filtered" "$DIFF_DUMP2" + fi + + log notice "generating diff" + + if diff --color=always -u "$DIFF_DUMP1" "$DIFF_DUMP2" \ + > "$DIFF_TMPDIR/diff" + then + log notice "no differences found" + else + log notice "differences found" + "$PAGER_OR_CAT" "$DIFF_TMPDIR/diff" + fi + + restore_namespace +} + +# ALLOWED_ACTIONS='backup,log' + +if ! [ "${AS_LIBRARY+x}" ]; then + if [ $# -eq 0 ]; then + help + exit 1 + fi + + if [ "$1" = '-h' ] || [ "$1" = '--help' ] || [ "$1" = '-?' ] || [ "$1" = 'help' ] || [ "$1" = '?' ]; then + help + exit 0 + fi + + while [ $# -gt 0 ]; do + case $1 in + -u|--url) + PGURL=$2 + DB_URL=$2 + shift 2 + ;; + deploy|replay|restore|patch|hydrate|backup|log|migrator|diff|pull_staging|test|check|cleanup|init) + if [ "${SUBCOMMAND+x}" ]; then + REMAINING_ARGS="$REMAINING_ARGS $(quote "$1")" + else + SUBCOMMAND="$1" + fi + shift + ;; + -i|--inherits) + INHERITS_LIST="${INHERITS_LIST+$INHERITS_LIST\"}$2" + shift 2 + ;; + *) REMAINING_ARGS="$REMAINING_ARGS $(quote "$1")"; shift ;; + esac + done + + [ "${SUBCOMMAND+x}" ] || ( log error "no subcommand specified. Use '--help' for usage information."; exit 1; ) + + # SAFETY: do not allow use on remote database + #if ! printf '%s\n' "$ALLOWED_ACTIONS" | grep "$SUBCOMMAND"; then + # IS_REMOTE="$(psql "$PGURL" -tAc "SELECT (inet_client_addr() IS NOT NULL)::int;")" + # [ "$IS_REMOTE" = 1 ] && (log error "THIS IS NOT LOCAL DATABSE, BASTARD"; exit 1) + #fi + + [ "${INHERITS_LIST+x}" ] && { + + INHERITS_LIST="$(printf '%s' "$INHERITS_LIST" | sed -E 's/"/,/g; s/([^,]+)/"\1"/g')" + + old_IFS="$IFS" + IFS=',' + check_inherits= + for table in $INHERITS_LIST; do + check_inherits="$(printf '%s\nSELECT 1 FROM %s LIMIT 1;' "$check_inherits" "$table")" + done + IFS="$old_IFS" + + check_inherits=$(printf '%s\n' \ + 'BEGIN;' \ + "$check_inherits" \ + 'COMMIT;') + + if ! db_exec "$check_inherits"; then + log error "init failed: ${WHITE}one of inherits table does not exists: ${CYAN}$INHERITS_LIST" + exit 5 + fi + } + + parsed_uri=$(mktemp) + trap 'rm -rf "$parsed_uri"' EXIT INT HUP + parse-uri "$PGURL" > "$parsed_uri" + # shellcheck disable=SC1090 + . "$parsed_uri" + + + [ "$REMAINING_ARGS" = '' ] || eval "set -- $REMAINING_ARGS"; REMAINING_ARGS= + "subcommand_$SUBCOMMAND" "$@" +fi diff --git a/package/db-tool/default.nix b/package/db-tool/default.nix new file mode 100644 index 0000000..8d71a2b --- /dev/null +++ b/package/db-tool/default.nix @@ -0,0 +1,62 @@ +{ dash, hectic, postgresql_17, neovim, openssh, coreutils, gawk, lib }: +let + shell = "${dash}/bin/dash"; + + database = hectic.writeShellApplication { + inherit shell; + bashOptions = [ + "errexit" + "nounset" + ]; + # SC2209: false positive — PAGER_OR_CAT=cat stores the string "cat" intentionally + excludeShellChecks = [ "SC2209" ]; + name = "database"; + runtimeInputs = [ hectic.migrator hectic.parse-uri postgresql_17 neovim openssh coreutils gawk ]; + + text = '' + ${builtins.readFile hectic.helpers.posix-shell.log} + ${builtins.readFile hectic.helpers.posix-shell.change_namespace} + ${builtins.readFile hectic.helpers.posix-shell.quote} + ${builtins.readFile hectic.helpers.posix-shell.pager_or_cat} + ${builtins.readFile ./database.sh} + ''; + + meta = { + description = "PostgreSQL development database management"; + mainProgram = "database"; + }; + }; + + postgresInit = hectic.writeShellApplication { + inherit shell; + bashOptions = [ ]; + name = "postgres-init"; + runtimeInputs = [ postgresql_17 coreutils ]; + + text = builtins.readFile ./postgres-init.sh; + + meta = { + description = "Initialize local PostgreSQL instance"; + mainProgram = "postgres-init"; + }; + }; + + postgresCleanup = hectic.writeShellApplication { + inherit shell; + bashOptions = [ ]; + name = "postgres-cleanup"; + runtimeInputs = [ postgresql_17 coreutils ]; + + text = builtins.readFile ./postgres-cleanup.sh; + + meta = { + description = "Clean up local PostgreSQL instance"; + mainProgram = "postgres-cleanup"; + }; + }; +in +{ + "db-tool" = database; + "postgres-init" = postgresInit; + "postgres-cleanup" = postgresCleanup; +} diff --git a/package/db-tool/postgres-cleanup.sh b/package/db-tool/postgres-cleanup.sh new file mode 100644 index 0000000..3faa82b --- /dev/null +++ b/package/db-tool/postgres-cleanup.sh @@ -0,0 +1,14 @@ +#!/bin/dash + +postgres_cleanup_main() { + if [ -z "${PG_WORKING_DIR:-}" ] && [ -z "${LOCAL_DIR:-}" ]; then return 0; fi + : "${PG_WORKING_DIR:=$LOCAL_DIR/focus/postgresql}" + if [ -f "${PG_WORKING_DIR}/data/postmaster.pid" ]; then + pg_ctl -D "${PG_WORKING_DIR}/data" -m fast -w stop || : + fi + return 0 +} + +if [ "$(basename "$0")" = 'postgres-cleanup' ]; then + postgres_cleanup_main "$@" +fi diff --git a/package/db-tool/postgres-init.sh b/package/db-tool/postgres-init.sh new file mode 100644 index 0000000..6824ebc --- /dev/null +++ b/package/db-tool/postgres-init.sh @@ -0,0 +1,49 @@ +#!/bin/dash + +postgres_init_main() { + if [ -z "${PG_WORKING_DIR:-}" ] && [ -z "${LOCAL_DIR:-}" ]; then + printf '%s\n' 'postgres-init: PG_WORKING_DIR or LOCAL_DIR is required' >&2 + return 1 + fi + + : "${PG_WORKING_DIR:=$LOCAL_DIR/focus/postgresql}" + : "${PG_PORT:=5432}" + : "${PG_DATABASE:=testdb}" + : "${PG_DISABLE_LOGGING:=0}" + [ "${PG_SHARED_PRELOAD_LIBRARIES+x}" ] || PG_SHARED_PRELOAD_LIBRARIES='pg_cron' + : "${PG_URL_VAR:=PGURL}" + + mkdir -p "$PG_WORKING_DIR" || return 1 + wd="$PG_WORKING_DIR"; data="$wd/data"; sockdir="$wd/sock"; db="$PG_DATABASE" + + pg_ctl -D "$data" -m fast -w stop >/dev/null 2>&1 || : + mkdir -p "$sockdir" || return 1 + + if [ "${PG_REUSE+x}" ] && [ -f "$data/PG_VERSION" ]; then PG_REUSE=1; else PG_REUSE=0; fi + if [ "$PG_REUSE" -eq 0 ]; then + rm -rf "$data" "$sockdir" || return 1 + mkdir -p "$sockdir" || return 1 + initdb -D "$data" --no-locale -E UTF8 || return 1 + { printf '%s\n' "listen_addresses = ''"; [ "$PG_DISABLE_LOGGING" -eq 0 ] && { printf '%s\n' 'logging_collector = on'; printf '%s\n' "log_directory = 'log'"; }; [ -n "$PG_SHARED_PRELOAD_LIBRARIES" ] && { printf '%s\n' "shared_preload_libraries = '$PG_SHARED_PRELOAD_LIBRARIES'"; printf '%s\n' "cron.database_name = '$db'"; printf '%s\n' "cron.host = '$sockdir'"; }; :; } >> "$data/postgresql.conf" || return 1 + sed -i "1ilocal all all trust" "$data/pg_hba.conf" || return 1 + fi + + sed -i '/^[[:space:]]*port[[:space:]]*=/d' "$data/postgresql.conf" || return 1 + sed -i '/^[[:space:]]*unix_socket_directories[[:space:]]*=/d' "$data/postgresql.conf" || return 1 + { printf '%s\n' "port = $PG_PORT"; printf '%s\n' "unix_socket_directories = '$sockdir'"; } >> "$data/postgresql.conf" || return 1 + pg_ctl -D "$data" -o "-F" -w start || return 2 + + user="$(id -un)" || return 1 + if [ "$PG_REUSE" -eq 0 ]; then createdb -h "$sockdir" -U "$user" "$db" || return 1; fi + psql -h "$sockdir" -p "$PG_PORT" -d "$db" -v ON_ERROR_STOP=1 -c 'select 1;' || return 1 + + export POSTGRESQL_HOST="$sockdir" POSTGRESQL_PORT="$PG_PORT" POSTGRESQL_USER="$user" POSTGRESQL_DATABASE="$db" + _pg_url="postgresql://${POSTGRESQL_USER}@/${POSTGRESQL_DATABASE}?host=${POSTGRESQL_HOST}&port=${POSTGRESQL_PORT}" + case $PG_URL_VAR in ''|*[!ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789_]* ) printf '%s\n' 'postgres-init: invalid PG_URL_VAR' >&2; return 1 ;; esac + export "${PG_URL_VAR}=${_pg_url}" || return 1 + return 0 +} + +if [ "$(basename "$0")" = 'postgres-init' ]; then + postgres_init_main "$@" +fi diff --git a/package/default.nix b/package/default.nix index 44ecb07..d946d61 100644 --- a/package/default.nix +++ b/package/default.nix @@ -107,6 +107,7 @@ }; nativeBuildInputs = with pkgs; [pkg-config curl]; }; + dbToolPkgs = pkgs.callPackage ./db-tool {}; in { py3-datetime = pkgs.callPackage ./py3-datetime.nix {}; py3-marzban = pkgs.callPackage ./py3-marzban.nix { inherit self; }; @@ -140,6 +141,10 @@ in { shellplot = pkgs.callPackage ./shellplot {}; onlinepubs2man = pkgs.callPackage ./onlinepubs2man {}; migrator = pkgs.callPackage ./migrator {}; + "parse-uri" = pkgs.callPackage ./parse-uri {}; + "db-tool" = dbToolPkgs."db-tool"; + "postgres-init" = dbToolPkgs."postgres-init"; + "postgres-cleanup" = dbToolPkgs."postgres-cleanup"; nbt2json = pkgs.callPackage ./nbt2json {}; hemar-parser = pkgs.callPackage ./hemar/parser {}; AstroTuxLauncher = pkgs.callPackage ./AstroTuxLauncher.nix {}; diff --git a/package/parse-uri/default.nix b/package/parse-uri/default.nix new file mode 100644 index 0000000..77afe0d --- /dev/null +++ b/package/parse-uri/default.nix @@ -0,0 +1,29 @@ +{ stdenv, gcc, libpq, lib, bash }: + +stdenv.mkDerivation { + pname = "parse-uri"; + version = "1.0"; + + src = ./.; + doCheck = false; + + nativeBuildInputs = [ gcc ]; + buildInputs = [ libpq ]; + + INCLUDES = "-I${libpq.dev}/include"; + LDFLAGS = "-L${libpq.out}/lib -lpq"; + + buildPhase = '' + ${bash}/bin/sh ./make.sh build + ''; + + installPhase = '' + mkdir -p $out/bin + cp target/parse-uri $out/bin/ + ''; + + meta = { + description = "parse-uri"; + license = lib.licenses.mit; + }; +} diff --git a/package/parse-uri/main.c b/package/parse-uri/main.c new file mode 100644 index 0000000..4ae7021 --- /dev/null +++ b/package/parse-uri/main.c @@ -0,0 +1,32 @@ +#include +#include +#include +#include + +int main(int argc, char *argv[]) { + if (argc != 2) { + printf("only 1 argument allow, please provide uri\n"); + exit(1); + } + const char *conninfo = argv[1]; + char *errmsg = NULL; + + PQconninfoOption *options = PQconninfoParse(conninfo, &errmsg); + if (!options) { + printf("Parse failed: %s\n", errmsg); + return 1; + } + + for (PQconninfoOption *opt = options; opt->keyword != NULL; opt++) { + char upper[128]; + size_t i = 0; + for (; opt->keyword[i] != '\0' && i < sizeof(upper)-1; i++) + upper[i] = toupper((unsigned char)opt->keyword[i]); + upper[i] = '\0'; + + printf("URI_%s=%s\n", upper, opt->val ? opt->val : ""); + } + + PQconninfoFree(options); + return 0; +} diff --git a/package/parse-uri/make.sh b/package/parse-uri/make.sh new file mode 100644 index 0000000..6d39667 --- /dev/null +++ b/package/parse-uri/make.sh @@ -0,0 +1,41 @@ +#!/bin/sh +# Usage: make.sh [build|check] [--norun] [--debug] [--color] + +PACKAGE_NAME="parse-uri" + +check_dependencies() { + for dep in cc; do + if ! command -v "$dep" >/dev/null 2>&1; then + echo "Error: Required dependency '$dep' not found." >&2 + exit 1 + fi + done +} +check_dependencies + +# Default flags +OPTFLAGS="-O2" +CFLAGS="-Wall -Wextra -Werror -pedantic" +STD_FLAGS="-std=c99" + +MODE="${1:-build}" +shift + +build() { + mkdir -p target + echo "# Build $PACKAGE_NAME" + # shellcheck disable=SC2086 + cc $CFLAGS $OPTFLAGS $STD_FLAGS main.c -o "target/$PACKAGE_NAME" $LDFLAGS $INCLUDES +} + +case "$MODE" in + build) + build + ;; + check) + echo "No tests to run" + ;; + *) + exit 1 + ;; +esac diff --git a/test/package/db-tool/default.nix b/test/package/db-tool/default.nix new file mode 100644 index 0000000..f2b1968 --- /dev/null +++ b/test/package/db-tool/default.nix @@ -0,0 +1,82 @@ +{ inputs, self, pkgs, system, ... }: let + lib = inputs.nixpkgs.lib; + + # turn anything under test directory into a derivation that exposes $out/run.sh + mkTestDrv = folder: name: type: + if type == "directory" then + pkgs.runCommand "test-${name}" {} '' + if ! [ -f ${"${folder}/${name}/run.sh"} ]; then + echo no run.sh in test/${name} + exit 1 + fi + + mkdir -p "$out" + cp -r ${"${folder}/${name}"}/* "$out/" + chmod +x "$out/run.sh" + '' + else if lib.hasSuffix ".sh" name then + pkgs.runCommand "test-${lib.removeSuffix ".sh" name}" {} '' + mkdir -p "$out" + install -Dm755 ${"${folder}/${name}"} "$out/run.sh" + '' + else + null; + + testDir = folder: builtins.readDir folder; + + # attrset: testName -> drv with run.sh + testDrvs = folder: + lib.mapAttrs' (n: v: + lib.nameValuePair (lib.removeSuffix ".sh" n) v + ) (lib.filterAttrs (_: v: v != null) + (lib.mapAttrs (n: t: mkTestDrv folder n t) (testDir folder))); + + database = self.packages.${system}."db-tool"; + postgresInit = self.packages.${system}."postgres-init"; + postgresCleanup = self.packages.${system}."postgres-cleanup"; + + # Non-postgres tests: .sh files at ./test/ (excluding postgresql/ subdir) + nonPgTestDrvs = + lib.mapAttrs' (n: v: lib.nameValuePair (lib.removeSuffix ".sh" n) v) + (lib.filterAttrs (_: v: v != null) + (lib.mapAttrs (n: t: mkTestDrv ./test n t) + (lib.filterAttrs (n: _: n != "postgresql") (testDir ./test)))); + + # Postgres tests: subdirs at ./test/postgresql/ + pgTestDrvs = testDrvs ./test/postgresql; + + mkNonPgTest = testName: testDrv: pkgs.runCommand "db-tool-${testName}" + { + nativeBuildInputs = [ pkgs.coreutils pkgs.gnugrep pkgs.gnused ]; + buildInputs = [ database postgresInit postgresCleanup pkgs.postgresql_17 pkgs.dash ]; + } '' + ${builtins.readFile self.legacyPackages.${system}.helpers.posix-shell.log} + test=${testDrv} + export HECTIC_LOG=trace + set -eu + + # shellcheck disable=SC1090 + . "$test/run.sh" + + mkdir -p "$out" + ''; + + mkPgTest = testName: testDrv: pkgs.runCommand "db-tool-${testName}" + { + nativeBuildInputs = [ pkgs.coreutils pkgs.gnugrep pkgs.gnused ]; + buildInputs = [ database postgresInit postgresCleanup pkgs.postgresql_17 pkgs.dash pkgs.netcat-openbsd ]; + } '' + ${builtins.readFile self.legacyPackages.${system}.helpers.posix-shell.log} + test=${testDrv} + export HECTIC_LOG=trace + set -eu + ${builtins.readFile ./postgresql/_lib.sh} + + # shellcheck disable=SC1090 + . "$test/run.sh" + + mkdir -p "$out" + ''; +in + (lib.mapAttrs (name: drv: mkNonPgTest name drv) nonPgTestDrvs) // + (lib.mapAttrs (name: drv: mkPgTest name drv) pgTestDrvs) diff --git a/test/package/db-tool/postgresql/_lib.sh b/test/package/db-tool/postgresql/_lib.sh new file mode 100644 index 0000000..2353c23 --- /dev/null +++ b/test/package/db-tool/postgresql/_lib.sh @@ -0,0 +1,147 @@ +# shellcheck shell=dash +# Shared PostgreSQL harness for db-tool tests. + +pg_harness__start_busy_socket() { + pg_harness_socket_dir="$1" + PG_HARNESS_BUSY_SOCKET_PATH="$pg_harness_socket_dir/.s.PGSQL.5432" + export PG_HARNESS_BUSY_SOCKET_PATH + + rm -f "$PG_HARNESS_BUSY_SOCKET_PATH" + nc -l -U "$PG_HARNESS_BUSY_SOCKET_PATH" >/dev/null 2>&1 & + PG_HARNESS_BUSY_SOCKET_PID=$! + export PG_HARNESS_BUSY_SOCKET_PID +} + +pg_harness_start() { + pg_harness_tmp_root="${TMPDIR:-/tmp}" + + if [ "${PG_HARNESS_PGDATA_OVERRIDE+x}" ]; then + pgdata_dir="$PG_HARNESS_PGDATA_OVERRIDE" + else + pgdata_dir=$(mktemp -d "$pg_harness_tmp_root/pgdata_XXXXXX") + fi + + PGDATA="$pgdata_dir" + export PGDATA + + trap 'pg_harness_stop' EXIT INT TERM + + if ! initdb -D "$pgdata_dir" --no-locale --encoding=UTF8 -U postgres >/dev/null 2>&1; then + pg_harness_stop + return 1 + fi + + if [ "${PG_HARNESS_INJECT_BUSY_SOCKET+x}" ] && ! [ "${PG_HARNESS_BUSY_SOCKET_PID+x}" ]; then + pg_harness__start_busy_socket "$pgdata_dir" + fi + + if ! pg_ctl start -D "$pgdata_dir" -o "-k $pgdata_dir -h ''" >/dev/null 2>&1; then + pg_harness_stop + return 1 + fi + + PGHOST="$pgdata_dir" + PGPORT="" + PGUSER="postgres" + PGDATABASE="postgres" + export PGHOST PGPORT PGUSER PGDATABASE + + POSTGRESQL_HOST="$PGHOST" + POSTGRESQL_PORT="$PGPORT" + POSTGRESQL_USER="$PGUSER" + POSTGRESQL_DATABASE="$PGDATABASE" + PGURL="postgresql://postgres@localhost/postgres?host=$pgdata_dir" + export POSTGRESQL_HOST POSTGRESQL_PORT POSTGRESQL_USER POSTGRESQL_DATABASE PGURL + + pg_harness_ready= + pg_harness_attempt=0 + while [ "$pg_harness_attempt" -lt 10 ]; do + if pg_isready -h "$pgdata_dir" >/dev/null 2>&1; then + pg_harness_ready=1 + break + fi + + sleep 0.5 + pg_harness_attempt=$((pg_harness_attempt + 1)) + done + + if ! [ "$pg_harness_ready" = 1 ]; then + pg_harness_stop + return 1 + fi +} + +pg_harness_stop() { + trap - EXIT INT TERM + + if [ "${PG_HARNESS_BUSY_SOCKET_PID+x}" ]; then + kill "$PG_HARNESS_BUSY_SOCKET_PID" >/dev/null 2>&1 || true + wait "$PG_HARNESS_BUSY_SOCKET_PID" >/dev/null 2>&1 || true + unset PG_HARNESS_BUSY_SOCKET_PID + fi + + if [ "${PG_HARNESS_BUSY_SOCKET_PATH+x}" ] && [ -n "$PG_HARNESS_BUSY_SOCKET_PATH" ]; then + rm -f "$PG_HARNESS_BUSY_SOCKET_PATH" + fi + + if [ "${PGDATA+x}" ] && [ -n "$PGDATA" ]; then + pg_ctl stop -D "$PGDATA" -m fast >/dev/null 2>&1 || true + rm -rf "$PGDATA" + fi + + if [ "${PG_HARNESS_PGDATA_OVERRIDE+x}" ] && [ -n "$PG_HARNESS_PGDATA_OVERRIDE" ]; then + if ! [ "${PGDATA+x}" ] || [ "$PG_HARNESS_PGDATA_OVERRIDE" != "$PGDATA" ]; then + rm -rf "$PG_HARNESS_PGDATA_OVERRIDE" + fi + fi + + if [ "${PG_HARNESS_CORRUPT_PATH+x}" ] && [ -n "$PG_HARNESS_CORRUPT_PATH" ]; then + rm -f "$PG_HARNESS_CORRUPT_PATH" + fi + + unset PG_HARNESS_BUSY_SOCKET_PATH + unset PG_HARNESS_CORRUPT_PATH + unset PG_HARNESS_INJECT_BUSY_SOCKET + unset PG_HARNESS_PGDATA_OVERRIDE + unset PGHOST PGPORT PGUSER PGDATABASE PGDATA + unset POSTGRESQL_HOST POSTGRESQL_PORT POSTGRESQL_USER POSTGRESQL_DATABASE PGURL +} + +pg_harness_start_corrupt_dir() { + pg_harness_tmp_root="${TMPDIR:-/tmp}" + PG_HARNESS_PGDATA_OVERRIDE=$(mktemp "$pg_harness_tmp_root/pgdata_corrupt_XXXXXX") + : > "$PG_HARNESS_PGDATA_OVERRIDE" + PG_HARNESS_CORRUPT_PATH="$PG_HARNESS_PGDATA_OVERRIDE" + export PG_HARNESS_PGDATA_OVERRIDE PG_HARNESS_CORRUPT_PATH +} + +pg_harness_busy_socket() { + pg_harness_tmp_root="${TMPDIR:-/tmp}" + PG_HARNESS_INJECT_BUSY_SOCKET=1 + export PG_HARNESS_INJECT_BUSY_SOCKET + + if [ "${PGDATA+x}" ] && [ -d "$PGDATA" ]; then + pg_harness_socket_dir="$PGDATA" + else + if [ "${PG_HARNESS_PGDATA_OVERRIDE+x}" ]; then + pg_harness_socket_dir="$PG_HARNESS_PGDATA_OVERRIDE" + else + pg_harness_socket_dir=$(mktemp -d "$pg_harness_tmp_root/pgdata_busy_XXXXXX") + PG_HARNESS_PGDATA_OVERRIDE="$pg_harness_socket_dir" + export PG_HARNESS_PGDATA_OVERRIDE + fi + fi + + if [ "${PGDATA+x}" ] && [ -d "$PGDATA" ] && ! [ "${PG_HARNESS_BUSY_SOCKET_PID+x}" ]; then + pg_harness__start_busy_socket "$pg_harness_socket_dir" + fi +} + +pg_harness_kill_postmaster() { + if ! [ "${PGDATA+x}" ] || ! [ -f "$PGDATA/postmaster.pid" ]; then + return 1 + fi + + IFS= read -r pg_harness_postmaster_pid < "$PGDATA/postmaster.pid" || return 1 + kill -KILL "$pg_harness_postmaster_pid" +} diff --git a/test/package/db-tool/test/help.sh b/test/package/db-tool/test/help.sh new file mode 100644 index 0000000..ed8b68d --- /dev/null +++ b/test/package/db-tool/test/help.sh @@ -0,0 +1,19 @@ +# shellcheck shell=dash + +HECTIC_NAMESPACE=test-db-tool-help + +log notice "test case: database --help exits 0" +if ! database --help > /tmp/help-out.txt 2>&1; then + log error "test failed: database --help exited non-zero" + exit 1 +fi + +for tok in deploy pull_staging cleanup check log init migrator; do + if ! grep -qF "$tok" /tmp/help-out.txt; then + log error "test failed: --help output missing token: $tok" + cat /tmp/help-out.txt >&2 + exit 1 + fi +done + +log notice "test passed" diff --git a/test/package/db-tool/test/missing-database-dir.sh b/test/package/db-tool/test/missing-database-dir.sh new file mode 100644 index 0000000..ba25c29 --- /dev/null +++ b/test/package/db-tool/test/missing-database-dir.sh @@ -0,0 +1,19 @@ +# shellcheck shell=dash + +HECTIC_NAMESPACE=test-db-tool-missing-dir + +export PGURL="" +unset LOCAL_DIR DATABASE_DIR DB_URL 2>/dev/null || true + +log notice "test case: database deploy fails without LOCAL_DIR" +set +e +database deploy 2>/tmp/missing-err.txt +code=$? +set -e + +if [ "$code" = 0 ]; then + log error "test failed: database deploy exited 0 without LOCAL_DIR" + exit 1 +fi + +log notice "test passed: exited $code" diff --git a/test/package/db-tool/test/postgresql/deploy-basic/run.sh b/test/package/db-tool/test/postgresql/deploy-basic/run.sh new file mode 100644 index 0000000..5747d15 --- /dev/null +++ b/test/package/db-tool/test/postgresql/deploy-basic/run.sh @@ -0,0 +1,19 @@ +# shellcheck shell=dash + +HECTIC_NAMESPACE=test-db-tool-deploy-basic + +pg_harness_start + +LOCAL_DIR=$(mktemp -d) +export LOCAL_DIR +mkdir -p "$LOCAL_DIR/devshell" +printf '#!/bin/dash\nexit 0\n' > "$LOCAL_DIR/devshell/postgres-init.sh" +chmod +x "$LOCAL_DIR/devshell/postgres-init.sh" + +log notice "test case: database deploy --no-hydrate --no-patch exits 0" +if ! database deploy --no-hydrate --no-patch; then + log error "database deploy failed" + exit 1 +fi + +log notice "test passed" diff --git a/test/package/db-tool/test/postgresql/deploy-cleanup-flag/run.sh b/test/package/db-tool/test/postgresql/deploy-cleanup-flag/run.sh new file mode 100644 index 0000000..c2a9c0e --- /dev/null +++ b/test/package/db-tool/test/postgresql/deploy-cleanup-flag/run.sh @@ -0,0 +1,20 @@ +# shellcheck shell=dash + +HECTIC_NAMESPACE=test-db-tool-deploy-cleanup-flag + +pg_harness_start + +LOCAL_DIR=$(mktemp -d) +export LOCAL_DIR +mkdir -p "$LOCAL_DIR/devshell" +printf '#!/bin/dash\nexit 0\n' > "$LOCAL_DIR/devshell/postgres-init.sh" +printf '#!/bin/dash\nexit 0\n' > "$LOCAL_DIR/devshell/postgres-cleanup.sh" +chmod +x "$LOCAL_DIR/devshell/postgres-init.sh" "$LOCAL_DIR/devshell/postgres-cleanup.sh" + +log notice "test case: database deploy --no-hydrate --no-patch --cleanup exits 0" +if ! database deploy --no-hydrate --no-patch --cleanup; then + log error "database deploy --cleanup failed" + exit 1 +fi + +log notice "test passed" diff --git a/test/package/db-tool/test/postgresql/init-cleanup-roundtrip/run.sh b/test/package/db-tool/test/postgresql/init-cleanup-roundtrip/run.sh new file mode 100644 index 0000000..1e51c64 --- /dev/null +++ b/test/package/db-tool/test/postgresql/init-cleanup-roundtrip/run.sh @@ -0,0 +1,36 @@ +# shellcheck shell=dash + +HECTIC_NAMESPACE=test-db-tool-init-cleanup-roundtrip + +PG_WORKING_DIR=$(mktemp -d) +export PG_WORKING_DIR PG_DATABASE=testdb PG_PORT=5432 PG_SHARED_PRELOAD_LIBRARIES='' + +cleanup() { + postgres-cleanup + rm -rf "$PG_WORKING_DIR" +} +trap 'cleanup' EXIT INT TERM + +log notice "test case: postgres-init starts cluster" +if ! postgres-init; then + log error "postgres-init failed" + exit 1 +fi + +pgurl="postgresql://$(id -un)@/testdb?host=${PG_WORKING_DIR}/sock&port=5432" +log notice "verifying connection" +if ! psql "$pgurl" -c 'SELECT 1;' >/dev/null 2>&1; then + log error "connection failed after postgres-init" + exit 1 +fi + +log notice "test case: postgres-cleanup stops cluster" +postgres-cleanup + +log notice "verifying cluster stopped" +if pg_isready -h "${PG_WORKING_DIR}/sock" -p 5432 >/dev/null 2>&1; then + log error "postgres still running after cleanup" + exit 1 +fi + +log notice "test passed" diff --git a/test/package/db-tool/test/postgresql/log-subcommand/run.sh b/test/package/db-tool/test/postgresql/log-subcommand/run.sh new file mode 100644 index 0000000..2f3b2bc --- /dev/null +++ b/test/package/db-tool/test/postgresql/log-subcommand/run.sh @@ -0,0 +1,18 @@ +# shellcheck shell=dash + +HECTIC_NAMESPACE=test-db-tool-log-subcommand + +PG_WORKING_DIR=$(mktemp -d) +LOCAL_DIR=$(mktemp -d) +export PG_WORKING_DIR LOCAL_DIR PGURL='postgresql://localhost/db' +mkdir -p "$PG_WORKING_DIR/data/log" + +trap 'rm -rf "$PG_WORKING_DIR" "$LOCAL_DIR"' EXIT INT TERM + +log notice "test case: database log list exits 0 with empty log dir" +if ! database log list; then + log error "database log list failed" + exit 1 +fi + +log notice "test passed" diff --git a/test/package/db-tool/test/postgresql/postgres-cleanup-stale-pidfile/run.sh b/test/package/db-tool/test/postgresql/postgres-cleanup-stale-pidfile/run.sh new file mode 100644 index 0000000..5fd2241 --- /dev/null +++ b/test/package/db-tool/test/postgresql/postgres-cleanup-stale-pidfile/run.sh @@ -0,0 +1,19 @@ +# shellcheck shell=dash + +HECTIC_NAMESPACE=test-db-tool-postgres-cleanup-stale-pidfile + +PG_WORKING_DIR=$(mktemp -d) +export PG_WORKING_DIR + +mkdir -p "$PG_WORKING_DIR/data" +printf '99999999\n' > "$PG_WORKING_DIR/data/postmaster.pid" + +trap 'rm -rf "$PG_WORKING_DIR"' EXIT INT TERM + +log notice "test case: postgres-cleanup exits 0 with stale pidfile" +if ! postgres-cleanup; then + log error "postgres-cleanup failed with stale pidfile" + exit 1 +fi + +log notice "test passed" diff --git a/test/package/db-tool/test/postgresql/postgres-init-busy-socket/run.sh b/test/package/db-tool/test/postgresql/postgres-init-busy-socket/run.sh new file mode 100644 index 0000000..fc575a2 --- /dev/null +++ b/test/package/db-tool/test/postgresql/postgres-init-busy-socket/run.sh @@ -0,0 +1,42 @@ +# shellcheck shell=dash + +HECTIC_NAMESPACE=test-db-tool-postgres-init-busy-socket + +PG_WORKING_DIR=$(mktemp -d) +export PG_WORKING_DIR PG_DATABASE=testdb PG_PORT=5432 PG_SHARED_PRELOAD_LIBRARIES='' + +trap 'pg_harness_stop; rm -rf "$PG_WORKING_DIR"' EXIT INT TERM + +log notice "setup: creating initial postgres cluster" +if ! postgres-init; then + log error "setup failed: initial postgres-init failed" + exit 1 +fi + +log notice "setup: stopping postgres to free socket" +postgres-cleanup + +log notice "setup: occupying socket with netcat" +pg_harness__start_busy_socket "$PG_WORKING_DIR/sock" +i=0 +while [ "$i" -lt 50 ] && ! [ -S "$PG_HARNESS_BUSY_SOCKET_PATH" ]; do + sleep 0.1 + i=$((i + 1)) +done +[ -S "$PG_HARNESS_BUSY_SOCKET_PATH" ] || { log error "busy socket not ready"; exit 1; } +printf '%d\n' "$PG_HARNESS_BUSY_SOCKET_PID" > "${PG_HARNESS_BUSY_SOCKET_PATH}.lock" + +log notice "test case: postgres-init fails when socket is pre-occupied" +PG_REUSE=1 +export PG_REUSE +set +e +postgres-init +code=$? +set -e + +if [ "$code" = 0 ]; then + log error "test failed: postgres-init exited 0 with busy socket" + exit 1 +fi + +log notice "test passed: exited $code" diff --git a/test/package/db-tool/test/postgresql/postgres-init-corrupt-pgdata/run.sh b/test/package/db-tool/test/postgresql/postgres-init-corrupt-pgdata/run.sh new file mode 100644 index 0000000..9a322e4 --- /dev/null +++ b/test/package/db-tool/test/postgresql/postgres-init-corrupt-pgdata/run.sh @@ -0,0 +1,23 @@ +# shellcheck shell=dash + +HECTIC_NAMESPACE=test-db-tool-postgres-init-corrupt-pgdata + +pg_harness_start_corrupt_dir + +export PG_WORKING_DIR="$PG_HARNESS_PGDATA_OVERRIDE" +export PG_SHARED_PRELOAD_LIBRARIES='' + +trap 'pg_harness_stop' EXIT INT TERM + +log notice "test case: postgres-init fails when PG_WORKING_DIR is a regular file" +set +e +postgres-init +code=$? +set -e + +if [ "$code" = 0 ]; then + log error "test failed: postgres-init exited 0 with corrupt dir" + exit 1 +fi + +log notice "test passed: exited $code" diff --git a/test/package/db-tool/test/pull-staging-no-env.sh b/test/package/db-tool/test/pull-staging-no-env.sh new file mode 100644 index 0000000..2576f71 --- /dev/null +++ b/test/package/db-tool/test/pull-staging-no-env.sh @@ -0,0 +1,26 @@ +# shellcheck shell=dash + +HECTIC_NAMESPACE=test-db-tool-pull-staging + +export PGURL="" +unset STAGING_SSH_HOST STAGING_DB_URL STAGING_USER STAGING_HOST 2>/dev/null || true + +log notice "test case: database pull_staging exits 3 without STAGING_SSH_HOST" +set +e +database pull_staging 2>/build/staging-err.txt +code=$? +set -e + +if [ "$code" != 3 ]; then + log error "test failed: expected exit 3, got $code" + exit 1 +fi + +if ! grep -q 'STAGING_SSH_HOST' /build/staging-err.txt; then + log error "test failed: stderr does not mention STAGING_SSH_HOST" + exit 1 +fi + +log notice "test passed" + +log notice "test passed" diff --git a/test/package/db-tool/test/unknown-subcommand.sh b/test/package/db-tool/test/unknown-subcommand.sh new file mode 100644 index 0000000..1d25174 --- /dev/null +++ b/test/package/db-tool/test/unknown-subcommand.sh @@ -0,0 +1,16 @@ +# shellcheck shell=dash + +HECTIC_NAMESPACE=test-db-tool-unknown + +log notice "test case: database nonsense exits non-zero (expected 1)" +set +e +database nonsense 2>/dev/null +code=$? +set -e + +if [ "$code" = 0 ]; then + log error "test failed: database nonsense exited 0 (should be non-zero)" + exit 1 +fi + +log notice "test passed: exited $code" diff --git a/test/package/default.nix b/test/package/default.nix index d0a2e4e..5a45b14 100644 --- a/test/package/default.nix +++ b/test/package/default.nix @@ -1,4 +1,5 @@ { system, inputs, self, pkgs }: (import ./migrator { inherit system inputs self pkgs; }) // (import ./hemar { inherit system inputs self pkgs; }) // - (import (./. + "/sentinèlla") { inherit system inputs self pkgs; }) + (import (./. + "/sentinèlla") { inherit system inputs self pkgs; }) // + (import ./db-tool { inherit system inputs self pkgs; })