diff --git a/lib/default.nix b/lib/default.nix index 6c07393f..e96f583b 100644 --- a/lib/default.nix +++ b/lib/default.nix @@ -103,7 +103,7 @@ in { # Consolidated SQL bundles for the `hectic` schema. Single source of truth # for everything that creates objects in the `hectic` namespace, used by - # migrator (init-time) and db-tool (postgres-init + hydrate). Consumers apply + # migrator (init-time), db-dev/database hydrate, and db-ops secrets loading. Consumers apply # the full bundle via lib/hook/apply-hectic-bundle.sh. # # The whole hectic system shares one `versionString`; `hectic-version.sql` diff --git a/lib/hook/apply-hectic-bundle.sh b/lib/hook/apply-hectic-bundle.sh index 3e1d5c05..222594b9 100644 --- a/lib/hook/apply-hectic-bundle.sh +++ b/lib/hook/apply-hectic-bundle.sh @@ -10,8 +10,8 @@ # Usage: # apply_hectic_bundle [] # -# If DOTENV_CONTENT is non-empty, it is loaded into hectic.secret via -# hectic.load_secrets_from_env() after the bundle is applied. +# If DOTENV_CONTENT is non-empty, it is base64-encoded and then loaded into +# hectic.secret via hectic.load_secrets_from_env() after the bundle is applied. # SQL file paths are substituted by Nix evaluation time. apply_hectic_bundle() { @@ -41,11 +41,9 @@ apply_hectic_bundle() { done if [ -n "$env_content" ]; then - # Dollar-quote with $ps_env$ tag to preserve all content verbatim. + env_content_b64="$(printf '%s' "$env_content" | base64 | tr -d '\n')" || return 1 psql "$pgurl" -v ON_ERROR_STOP=1 <)` if `HECTIC_DOTENV_FILE` (or `${LOCAL_DIR}/.env.${ENVIRONMENT}`) is readable. | +| `db-ops secrets load` | **Yes, mandatory.** Applies the bundle and requires a dotenv source before loading secrets into `hectic.secret`. | The bundle is idempotent — repeated application is safe. diff --git a/package/db-tool/db-ops.sh b/package/db-tool/db-ops.sh new file mode 100644 index 00000000..d609a769 --- /dev/null +++ b/package/db-tool/db-ops.sh @@ -0,0 +1,222 @@ +# shellcheck shell=dash + +: "${SCRIPT_NAME:=$(basename "$0")}" + +help() { + # shellcheck disable=SC2059 + printf "$(cat < [OPTIONS] + +PostgreSQL operations utility. + +${BGREEN}Global Options:${NC} + ${BCYAN}-h${NC}, ${BCYAN}--help${NC} Show this help message + ${BCYAN}-u${NC}, ${BCYAN}--url${NC} PostgreSQL connection string + +${BGREEN}Subcommands:${NC} + ${BCYAN}secrets load${NC} [OPTIONS] Apply hectic bundle and load secrets + +${BGREEN}Environment:${NC} + ${BBLACK}PGURL${NC} PostgreSQL connection string fallback + ${BBLACK}DB_URL${NC} PostgreSQL connection string fallback + ${BBLACK}HECTIC_DOTENV_CONTENT${NC} Raw dotenv content to load + ${BBLACK}HECTIC_DOTENV_FILE${NC} Dotenv file to read when readable + ${BBLACK}LOCAL_DIR${NC} Used with ENVIRONMENT for .env fallback + ${BBLACK}ENVIRONMENT${NC} Falls back to ${BBLACK}\$LOCAL_DIR/.env.\$ENVIRONMENT${NC} + +EOF +)" +} + +help_secrets_load() { + # shellcheck disable=SC2059 + printf "$(cat < PostgreSQL connection string + ${BCYAN}-f${NC}, ${BCYAN}--dotenv-file${NC} Dotenv file path + +${BGREEN}Resolution order:${NC} + 1. ${BBLACK}HECTIC_DOTENV_CONTENT${NC} + 2. ${BBLACK}--dotenv-file${NC} / ${BBLACK}HECTIC_DOTENV_FILE${NC} + 3. ${BBLACK}\$LOCAL_DIR/.env.\$ENVIRONMENT${NC} + +Fails with exit code ${BBLACK}3${NC} when neither a PostgreSQL URL nor a dotenv +source can be resolved. + +EOF +)" +} + +resolve_pgurl() { + if [ -n "${PGURL:-}" ]; then + printf '%s' "$PGURL" + elif [ -n "${DB_URL:-}" ]; then + printf '%s' "$DB_URL" + else + return 1 + fi +} + +resolve_dotenv_content() { + dotenv_file="${1:-}" + + if [ -n "${HECTIC_DOTENV_CONTENT:-}" ]; then + printf '%s' "$HECTIC_DOTENV_CONTENT" + elif [ -n "$dotenv_file" ]; then + if [ ! -f "$dotenv_file" ] || [ ! -r "$dotenv_file" ]; then + return 2 + fi + cat "$dotenv_file" + elif [ -n "${ENVIRONMENT:-}" ] && [ -n "${LOCAL_DIR:-}" ] && [ -r "${LOCAL_DIR}/.env.${ENVIRONMENT}" ]; then + cat "${LOCAL_DIR}/.env.${ENVIRONMENT}" + else + return 1 + fi +} + +subcommand_secrets_load() { + change_namespace 'db ops secrets load' + + dotenv_file="${HECTIC_DOTENV_FILE:-}" + + while [ $# -gt 0 ]; do + case $1 in + -h|--help) + help_secrets_load + restore_namespace + exit 0 + ;; + -u|--url) + if [ $# -lt 2 ]; then + log error "missing value for $1" + restore_namespace + exit 3 + fi + PGURL=$2 + DB_URL=$2 + shift 2 + ;; + -f|--dotenv-file) + if [ $# -lt 2 ]; then + log error "missing value for $1" + restore_namespace + exit 3 + fi + dotenv_file=$2 + shift 2 + ;; + --*|-*) + log error "secrets load argument $1 does not exist" + restore_namespace + exit 9 + ;; + *) + log error "secrets load subcommand $1 does not exist" + restore_namespace + exit 9 + ;; + esac + done + + if ! pgurl="$(resolve_pgurl)"; then + log error "PGURL or DB_URL is required" + restore_namespace + exit 3 + fi + + if dotenv_content="$(resolve_dotenv_content "$dotenv_file")"; then + : + else + dotenv_content_exit_code=$? + if [ "$dotenv_content_exit_code" -eq 2 ]; then + log error "dotenv file is not readable: $dotenv_file" + else + log error "dotenv source is required (HECTIC_DOTENV_CONTENT, readable dotenv file, or \$LOCAL_DIR/.env.\$ENVIRONMENT)" + fi + restore_namespace + exit 3 + fi + + log notice "apply hectic bundle and load secrets" + apply_hectic_bundle "$pgurl" "$dotenv_content" + restore_namespace +} + +subcommand_secrets() { + change_namespace 'db ops secrets' + + if [ $# -eq 0 ]; then + help_secrets_load + restore_namespace + exit 0 + fi + + subcommand=$1 + shift + + case $subcommand in + load) + subcommand_secrets_load "$@" + ;; + -h|--help) + help_secrets_load + restore_namespace + exit 0 + ;; + *) + log error "secrets subcommand $subcommand does not exist" + restore_namespace + exit 9 + ;; + esac +} + +while [ $# -gt 0 ]; do + case $1 in + -h|--help) + help + exit 0 + ;; + -u|--url) + if [ $# -lt 2 ]; then + log error "missing value for $1" + exit 3 + fi + PGURL=$2 + DB_URL=$2 + shift 2 + ;; + --*|-*) + log error "argument $1 does not exist" + exit 9 + ;; + *) + break + ;; + esac +done + +subcommand="${1:-}" + +if [ -z "$subcommand" ]; then + help + exit 0 +fi + +shift + +case $subcommand in + secrets) + subcommand_secrets "$@" + ;; + *) + log error "subcommand $subcommand does not exist" + exit 9 + ;; +esac diff --git a/package/db-tool/default.nix b/package/db-tool/default.nix index 9be4f4ab..5d3b808b 100644 --- a/package/db-tool/default.nix +++ b/package/db-tool/default.nix @@ -8,7 +8,7 @@ let applyBundle = self.lib.hectic.applyBundleScript; - mkDatabase = + mkDbDev = { postgresql ? postgresql_17 }: hectic.writeShellApplication { inherit shell; @@ -36,6 +36,31 @@ let }; }; + mkDbOps = + { postgresql ? postgresql_17 }: + hectic.writeShellApplication { + inherit shell; + bashOptions = [ + "errexit" + "nounset" + ]; + excludeShellChecks = [ "SC2209" ]; + name = "db-ops"; + runtimeInputs = [ postgresql coreutils ]; + + text = '' + ${builtins.readFile hectic.helpers.posix-shell.log} + ${builtins.readFile hectic.helpers.posix-shell.change_namespace} + ${applyBundle} + ${builtins.readFile ./db-ops.sh} + ''; + + meta = { + description = "PostgreSQL operations utility"; + mainProgram = "db-ops"; + }; + }; + mkPostgresInit = { postgresql ? postgresql_17 }: hectic.writeShellApplication { @@ -73,7 +98,9 @@ let }; in { - "db-tool" = lib.makeOverridable mkDatabase { }; + "db-dev" = lib.makeOverridable mkDbDev { }; + "db-ops" = lib.makeOverridable mkDbOps { }; + "db-tool" = lib.makeOverridable mkDbDev { }; "postgres-init" = lib.makeOverridable mkPostgresInit { }; "postgres-cleanup" = lib.makeOverridable mkPostgresCleanup { }; "hectic-inheritance" = hecticInheritance; diff --git a/package/default.nix b/package/default.nix index 9cb66f46..14dfe24f 100644 --- a/package/default.nix +++ b/package/default.nix @@ -151,6 +151,8 @@ in { onlinepubs2man = pkgs.callPackage ./onlinepubs2man {}; migrator = pkgs.callPackage ./migrator { inherit self; }; "parse-uri" = pkgs.callPackage ./parse-uri {}; + "db-dev" = dbToolPkgs."db-dev"; + "db-ops" = dbToolPkgs."db-ops"; "db-tool" = dbToolPkgs."db-tool"; "postgres-init" = dbToolPkgs."postgres-init"; "postgres-cleanup" = dbToolPkgs."postgres-cleanup"; diff --git a/package/migrator/default.nix b/package/migrator/default.nix index 1d51942c..81df4873 100644 --- a/package/migrator/default.nix +++ b/package/migrator/default.nix @@ -1,4 +1,4 @@ -{ dash, hectic, sqlite, postgresql_17, gawk, self }: +{ dash, hectic, sqlite, postgresql_17, gawk, coreutils, self }: let shell = "${dash}/bin/dash"; bashOptions = [ @@ -11,7 +11,7 @@ let migrator = hectic.writeShellApplication { inherit shell bashOptions; name = "migrator"; - runtimeInputs = [ sqlite postgresql_17 gawk ]; + runtimeInputs = [ sqlite postgresql_17 gawk coreutils ]; text = '' ${builtins.readFile hectic.helpers.posix-shell.log} diff --git a/test/package/db-tool/default.nix b/test/package/db-tool/default.nix index f2b19686..0ed03998 100644 --- a/test/package/db-tool/default.nix +++ b/test/package/db-tool/default.nix @@ -31,7 +31,8 @@ ) (lib.filterAttrs (_: v: v != null) (lib.mapAttrs (n: t: mkTestDrv folder n t) (testDir folder))); - database = self.packages.${system}."db-tool"; + database = self.packages.${system}."db-dev"; + dbOps = self.packages.${system}."db-ops"; postgresInit = self.packages.${system}."postgres-init"; postgresCleanup = self.packages.${system}."postgres-cleanup"; @@ -48,7 +49,7 @@ mkNonPgTest = testName: testDrv: pkgs.runCommand "db-tool-${testName}" { nativeBuildInputs = [ pkgs.coreutils pkgs.gnugrep pkgs.gnused ]; - buildInputs = [ database postgresInit postgresCleanup pkgs.postgresql_17 pkgs.dash ]; + buildInputs = [ database dbOps postgresInit postgresCleanup pkgs.postgresql_17 pkgs.dash ]; } '' ${builtins.readFile self.legacyPackages.${system}.helpers.posix-shell.log} test=${testDrv} @@ -64,7 +65,7 @@ mkPgTest = testName: testDrv: pkgs.runCommand "db-tool-${testName}" { nativeBuildInputs = [ pkgs.coreutils pkgs.gnugrep pkgs.gnused ]; - buildInputs = [ database postgresInit postgresCleanup pkgs.postgresql_17 pkgs.dash pkgs.netcat-openbsd ]; + buildInputs = [ database dbOps postgresInit postgresCleanup pkgs.postgresql_17 pkgs.dash pkgs.netcat-openbsd ]; } '' ${builtins.readFile self.legacyPackages.${system}.helpers.posix-shell.log} test=${testDrv} diff --git a/test/package/db-tool/test/db-ops-help.sh b/test/package/db-tool/test/db-ops-help.sh new file mode 100644 index 00000000..9be5882e --- /dev/null +++ b/test/package/db-tool/test/db-ops-help.sh @@ -0,0 +1,19 @@ +# shellcheck shell=dash + +export HECTIC_NAMESPACE=test-db-ops-help + +log notice "test case: db-ops --help exits 0" +if ! db-ops --help > /tmp/db-ops-help-out.txt 2>&1; then + log error "test failed: db-ops --help exited non-zero" + exit 1 +fi + +for tok in secrets load HECTIC_DOTENV_FILE PGURL DB_URL; do + if ! grep -qF "$tok" /tmp/db-ops-help-out.txt; then + log error "test failed: db-ops --help output missing token: $tok" + cat /tmp/db-ops-help-out.txt >&2 + exit 1 + fi +done + +log notice "test passed" diff --git a/test/package/db-tool/test/db-ops-missing-dotenv.sh b/test/package/db-tool/test/db-ops-missing-dotenv.sh new file mode 100644 index 00000000..0cb9a4fb --- /dev/null +++ b/test/package/db-tool/test/db-ops-missing-dotenv.sh @@ -0,0 +1,24 @@ +# shellcheck shell=dash + +export HECTIC_NAMESPACE=test-db-ops-missing-dotenv + +if db-ops --url 'postgresql://example.invalid/test' secrets load > /tmp/db-ops-missing-dotenv.out 2>&1; then + log error "expected db-ops secrets load to fail without dotenv source" + exit 1 +else + exit_code=$? +fi + +[ "$exit_code" -eq 3 ] || { + log error "expected exit code 3 without dotenv source, got: $exit_code" + cat /tmp/db-ops-missing-dotenv.out >&2 + exit 1 +} + +if ! grep -qF 'dotenv source is required' /tmp/db-ops-missing-dotenv.out; then + log error "missing dotenv source error message" + cat /tmp/db-ops-missing-dotenv.out >&2 + exit 1 +fi + +log notice "test passed" diff --git a/test/package/db-tool/test/db-ops-missing-pgurl.sh b/test/package/db-tool/test/db-ops-missing-pgurl.sh new file mode 100644 index 00000000..074249e6 --- /dev/null +++ b/test/package/db-tool/test/db-ops-missing-pgurl.sh @@ -0,0 +1,28 @@ +# shellcheck shell=dash + +export HECTIC_NAMESPACE=test-db-ops-missing-pgurl + +dotenv_file=$(mktemp) +trap 'rm -f "$dotenv_file"' EXIT INT TERM +printf 'TEST_SECRET=hello-world\n' > "$dotenv_file" + +if db-ops secrets load --dotenv-file "$dotenv_file" > /tmp/db-ops-missing-pgurl.out 2>&1; then + log error "expected db-ops secrets load to fail without PGURL" + exit 1 +else + exit_code=$? +fi + +[ "$exit_code" -eq 3 ] || { + log error "expected exit code 3 without PGURL, got: $exit_code" + cat /tmp/db-ops-missing-pgurl.out >&2 + exit 1 +} + +if ! grep -qF 'PGURL or DB_URL is required' /tmp/db-ops-missing-pgurl.out; then + log error "missing PGURL error message" + cat /tmp/db-ops-missing-pgurl.out >&2 + exit 1 +fi + +log notice "test passed" diff --git a/test/package/db-tool/test/db-ops-unreadable-dotenv.sh b/test/package/db-tool/test/db-ops-unreadable-dotenv.sh new file mode 100644 index 00000000..d1a9b248 --- /dev/null +++ b/test/package/db-tool/test/db-ops-unreadable-dotenv.sh @@ -0,0 +1,28 @@ +# shellcheck shell=dash + +export HECTIC_NAMESPACE=test-db-ops-unreadable-dotenv + +dotenv_dir=$(mktemp -d) +dotenv_file="$dotenv_dir/missing.env" +trap 'rm -rf "$dotenv_dir"' EXIT INT TERM + +if db-ops --url 'postgresql://example.invalid/test' secrets load --dotenv-file "$dotenv_file" > /tmp/db-ops-unreadable-dotenv.out 2>&1; then + log error "expected db-ops secrets load to fail for unreadable explicit dotenv path" + exit 1 +else + exit_code=$? +fi + +[ "$exit_code" -eq 3 ] || { + log error "expected exit code 3 for unreadable explicit dotenv path, got: $exit_code" + cat /tmp/db-ops-unreadable-dotenv.out >&2 + exit 1 +} + +if ! grep -qF 'dotenv file is not readable' /tmp/db-ops-unreadable-dotenv.out; then + log error "missing unreadable dotenv file error message" + cat /tmp/db-ops-unreadable-dotenv.out >&2 + exit 1 +fi + +log notice "test passed" diff --git a/test/package/db-tool/test/postgresql/db-ops-secrets-load/run.sh b/test/package/db-tool/test/postgresql/db-ops-secrets-load/run.sh new file mode 100644 index 00000000..616e202e --- /dev/null +++ b/test/package/db-tool/test/postgresql/db-ops-secrets-load/run.sh @@ -0,0 +1,56 @@ +# shellcheck shell=dash + +HECTIC_NAMESPACE=test-db-ops-secrets-load + +PG_WORKING_DIR=$(mktemp -d) +export PG_WORKING_DIR PG_DATABASE=testdb PG_PORT=5432 PG_SHARED_PRELOAD_LIBRARIES='' + +cleanup() { + postgres-cleanup >/dev/null 2>&1 || : + rm -rf "$PG_WORKING_DIR" +} +trap 'cleanup' EXIT INT TERM + +if ! postgres-init; then + log error "postgres-init failed" + exit 1 +fi + +PGURL="postgresql://$(id -un)@/testdb?host=${PG_WORKING_DIR}/sock&port=5432" +export PGURL + +dotenv_file=$(mktemp) +printf 'TEST_SECRET=hello-world\nQUOTED_SECRET="quoted value"\nDELIMITER_SECRET=before$ps_env$after\n' > "$dotenv_file" + +if ! db-ops secrets load --dotenv-file "$dotenv_file"; then + log error "db-ops secrets load failed" + rm -f "$dotenv_file" + exit 1 +fi +rm -f "$dotenv_file" + +secret_value=$(psql "$PGURL" -v ON_ERROR_STOP=1 -tAc "SELECT value FROM hectic.secret WHERE key = 'TEST_SECRET';") || exit 1 +[ "$secret_value" = "hello-world" ] || { + log error "expected TEST_SECRET to be loaded, got: $secret_value" + exit 1 +} + +quoted_value=$(psql "$PGURL" -v ON_ERROR_STOP=1 -tAc "SELECT value FROM hectic.secret WHERE key = 'QUOTED_SECRET';") || exit 1 +[ "$quoted_value" = "quoted value" ] || { + log error "expected QUOTED_SECRET to strip outer quotes, got: $quoted_value" + exit 1 +} + +delimiter_value=$(psql "$PGURL" -v ON_ERROR_STOP=1 -tAc "SELECT value FROM hectic.secret WHERE key = 'DELIMITER_SECRET';") || exit 1 +[ "$delimiter_value" = 'before$ps_env$after' ] || { + log error "expected DELIMITER_SECRET to preserve dollar-quote delimiter text, got: $delimiter_value" + exit 1 +} + +hectic_schema=$(psql "$PGURL" -v ON_ERROR_STOP=1 -tAc "SELECT count(*) FROM pg_namespace WHERE nspname='hectic';") || exit 1 +[ "$hectic_schema" = 1 ] || { + log error "expected hectic schema to exist, got: $hectic_schema" + exit 1 +} + +log notice "test passed"