hectic-lab/util.nix
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: postgres hooks

Browse Source
This commit is contained in:
yukkop
2026-04-30 21:59:53 +00:00
parent bf7ee34716
commit 3d5e3fdb36
8 changed files with 127 additions and 132 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
lib/default.nix
View File

@@ -102,7 +102,8 @@ in {
# Consolidated SQL bundles for the `hectic` schema. Single source of truth
# for everything that creates objects in the `hectic` namespace, used by
# migrator (init-time), db-tool (postgres-init), and pkgs.hectic.postgres-secrets.
# migrator (init-time) and db-tool (postgres-init + hydrate). Consumers apply
# the full bundle via lib/hook/apply-hectic-bundle.sh.
#
# The whole hectic system shares one `versionString`; `hectic-version.sql`
# registers (`'hectic'`, versionString) into `hectic.version` and raises an
@@ -126,6 +127,7 @@ in {
secret = static ./hook/sql/hectic-secret.sql;
migration = static ./hook/sql/hectic-migration.sql;
inheritance = static ./hook/sql/hectic-inheritance.sql;
applyBundleScript = ./hook/apply-hectic-bundle.sh;
};
# Back-compat alias. Prefer `self.lib.hectic.inheritance`.

58
lib/hook/apply-hectic-bundle.sh Normal file
View File

@@ -0,0 +1,58 @@
#!/bin/dash
# Applies the full hectic SQL bundle to a PostgreSQL database, in order:
# 1. version (hard-fails on version mismatch)
# 2. secret (hectic.secret table + load_secrets_from_env + get_secret)
# 3. migration (hectic.migration table + domains + sha256_lower trigger)
# 4. inheritance (created_at/updated_at/immutable enforcement triggers)
#
# Idempotent: each SQL file uses IF NOT EXISTS / CREATE OR REPLACE.
#
# Required env (caller injects from Nix):
# HECTIC_VERSION_SQL - path to hectic-version.sql (substituted)
# HECTIC_SECRET_SQL - path to hectic-secret.sql
# HECTIC_MIGRATION_SQL - path to hectic-migration.sql
# HECTIC_INHERITANCE_SQL - path to hectic-inheritance.sql
#
# Usage:
# apply_hectic_bundle <PGURL> [<DOTENV_CONTENT>]
#
# If DOTENV_CONTENT is non-empty, it is loaded into hectic.secret via
# hectic.load_secrets_from_env() after the bundle is applied.
apply_hectic_bundle() {
pgurl="${1:-}"
env_content="${2:-}"
if [ -z "$pgurl" ]; then
printf '%s\n' 'apply-hectic-bundle: PGURL is required (arg 1)' >&2
return 3
fi
for var in HECTIC_VERSION_SQL HECTIC_SECRET_SQL HECTIC_MIGRATION_SQL HECTIC_INHERITANCE_SQL; do
eval "val=\${$var:-}"
if [ -z "$val" ]; then
printf '%s\n' "apply-hectic-bundle: $var not set" >&2
return 3
fi
if [ ! -r "$val" ]; then
printf '%s\n' "apply-hectic-bundle: $var not readable: $val" >&2
return 1
fi
done
psql "$pgurl" -v ON_ERROR_STOP=1 -f "$HECTIC_VERSION_SQL" || return 1
psql "$pgurl" -v ON_ERROR_STOP=1 -f "$HECTIC_SECRET_SQL" || return 1
psql "$pgurl" -v ON_ERROR_STOP=1 -f "$HECTIC_MIGRATION_SQL" || return 1
psql "$pgurl" -v ON_ERROR_STOP=1 -f "$HECTIC_INHERITANCE_SQL" || return 1
if [ -n "$env_content" ]; then
# Dollar-quote with $ps_env$ tag to preserve all content verbatim.
psql "$pgurl" -v ON_ERROR_STOP=1 <<SQL || return 1
SELECT hectic.load_secrets_from_env(\$ps_env\$
$env_content
\$ps_env\$);
SQL
fi
return 0
}

12
package/db-tool/database.sh
View File

@@ -942,15 +942,21 @@ subcommand_hydrate() {
done
if [ ! "${HYDRATE_NO_HOOK+x}" ]; then
log info "hectic secrets hook"
log info "hectic bundle hook"
# shellcheck disable=SC2059
printf "${BBLACK}"
sh "${LOCAL_DIR}/lib/hook/postgres-secrets.sh" "$PGURL" "$ENVIRONMENT"
dotenv_content=""
if [ -n "${HECTIC_DOTENV_FILE:-}" ] && [ -r "$HECTIC_DOTENV_FILE" ]; then
dotenv_content="$(cat "$HECTIC_DOTENV_FILE")"
elif [ -n "${ENVIRONMENT:-}" ] && [ -r "${LOCAL_DIR}/.env.${ENVIRONMENT}" ]; then
dotenv_content="$(cat "${LOCAL_DIR}/.env.${ENVIRONMENT}")"
fi
apply_hectic_bundle "$PGURL" "$dotenv_content"
# shellcheck disable=SC2059
printf "${NC}"
else
log info "skipping hectic secrets hook"
log info "skipping hectic bundle hook"
fi
local mock_arg=""

28
package/db-tool/default.nix
View File

@@ -1,4 +1,4 @@
{ dash, hectic, postgresql_17, neovim, openssh, coreutils, gawk, lib, runCommand }:
{ dash, hectic, postgresql_17, neovim, openssh, coreutils, gawk, lib, runCommand, self }:
let
shell = "${dash}/bin/dash";
@@ -9,6 +9,23 @@ let
cp ${hecticInheritanceSqlPath} "$out/share/hectic/hectic-inheritance.sql"
'';
# Materialize the templated version SQL into the Nix store as a real file
# so it can be passed by path to psql -f (alongside the static siblings).
hecticVersionSqlFile = pkgs-writeText "hectic-version.sql" self.lib.hectic.version.sql;
pkgs-writeText = name: text: runCommand name { inherit text; passAsFile = [ "text" ]; } ''
cp "$textPath" "$out"
'';
hecticEnv = ''
HECTIC_VERSION_SQL=${hecticVersionSqlFile}
HECTIC_SECRET_SQL=${self.lib.hectic.secret.path}
HECTIC_MIGRATION_SQL=${self.lib.hectic.migration.path}
HECTIC_INHERITANCE_SQL=${self.lib.hectic.inheritance.path}
export HECTIC_VERSION_SQL HECTIC_SECRET_SQL HECTIC_MIGRATION_SQL HECTIC_INHERITANCE_SQL
'';
applyBundle = builtins.readFile self.lib.hectic.applyBundleScript;
mkDatabase =
{ postgresql ? postgresql_17 }:
hectic.writeShellApplication {
@@ -17,7 +34,6 @@ let
"errexit"
"nounset"
];
# SC2209: false positive — PAGER_OR_CAT=cat stores the string "cat" intentionally
excludeShellChecks = [ "SC2209" ];
name = "database";
runtimeInputs = [ hectic.migrator hectic.parse-uri postgresql neovim openssh coreutils gawk ];
@@ -27,6 +43,8 @@ let
${builtins.readFile hectic.helpers.posix-shell.change_namespace}
${builtins.readFile hectic.helpers.posix-shell.quote}
${builtins.readFile hectic.helpers.posix-shell.pager_or_cat}
${hecticEnv}
${applyBundle}
${builtins.readFile ./database.sh}
'';
@@ -44,11 +62,7 @@ let
name = "postgres-init";
runtimeInputs = [ postgresql coreutils ];
text = ''
HECTIC_INHERITANCE_SQL_DEFAULT="${hecticInheritance}/share/hectic/hectic-inheritance.sql"
export HECTIC_INHERITANCE_SQL_DEFAULT
${builtins.readFile ./postgres-init.sh}
'';
text = builtins.readFile ./postgres-init.sh;
meta = {
description = "Initialize local PostgreSQL instance";

10
package/db-tool/postgres-init.sh
View File

@@ -48,16 +48,6 @@ postgres_init_main() {
fi
psql -h "$sockdir" -p "$PG_PORT" -d "$db" -v ON_ERROR_STOP=1 -c 'select 1;' || return 1
if [ "${PG_HECTIC_INHERITANCE:-1}" = "1" ]; then
sql_file="${HECTIC_INHERITANCE_SQL:-${HECTIC_INHERITANCE_SQL_DEFAULT:-}}"
if [ -z "$sql_file" ]; then
printf '%s\n' 'postgres-init: PG_HECTIC_INHERITANCE=1 but no SQL file resolved (set HECTIC_INHERITANCE_SQL)' >&2
return 3
fi
[ -r "$sql_file" ] || { printf '%s\n' "postgres-init: hectic-inheritance SQL not readable: $sql_file" >&2; return 1; }
psql -h "$sockdir" -p "$PG_PORT" -U "$user" -d "$db" -v ON_ERROR_STOP=1 -f "$sql_file" || return 1
fi
export POSTGRESQL_HOST="$sockdir" POSTGRESQL_PORT="$PG_PORT" POSTGRESQL_USER="$user" POSTGRESQL_DATABASE="$db"
_pg_url="postgresql://${POSTGRESQL_USER}@/${POSTGRESQL_DATABASE}?host=${POSTGRESQL_HOST}&port=${POSTGRESQL_PORT}"
case $PG_URL_VAR in ''|*[!ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789_]* ) printf '%s\n' 'postgres-init: invalid PG_URL_VAR' >&2; return 1 ;; esac

4
package/default.nix
View File

@@ -107,7 +107,7 @@
};
nativeBuildInputs = with pkgs; [pkg-config curl];
};
dbToolPkgs = pkgs.callPackage ./db-tool {};
dbToolPkgs = pkgs.callPackage ./db-tool { inherit self; };
in {
py3-datetime = pkgs.callPackage ./py3-datetime.nix {};
py3-marzban = pkgs.callPackage ./py3-marzban.nix { inherit self; };
@@ -140,7 +140,7 @@ in {
deploy = pkgs.callPackage ./deploy { inherit inputs; };
shellplot = pkgs.callPackage ./shellplot {};
onlinepubs2man = pkgs.callPackage ./onlinepubs2man {};
migrator = pkgs.callPackage ./migrator {};
migrator = pkgs.callPackage ./migrator { inherit self; };
"parse-uri" = pkgs.callPackage ./parse-uri {};
"db-tool" = dbToolPkgs."db-tool";
"postgres-init" = dbToolPkgs."postgres-init";

19
package/migrator/default.nix
View File

@@ -1,4 +1,4 @@
{ dash, hectic, sqlite, postgresql_17, gawk }:
{ dash, hectic, sqlite, postgresql_17, gawk, runCommand, self }:
let
shell = "${dash}/bin/dash";
bashOptions = [
@@ -6,6 +6,21 @@ let
"nounset"
];
hecticVersionSqlFile = runCommand "hectic-version.sql" {
text = self.lib.hectic.version.sql;
passAsFile = [ "text" ];
} ''cp "$textPath" "$out"'';
hecticEnv = ''
HECTIC_VERSION_SQL=${hecticVersionSqlFile}
HECTIC_SECRET_SQL=${self.lib.hectic.secret.path}
HECTIC_MIGRATION_SQL=${self.lib.hectic.migration.path}
HECTIC_INHERITANCE_SQL=${self.lib.hectic.inheritance.path}
export HECTIC_VERSION_SQL HECTIC_SECRET_SQL HECTIC_MIGRATION_SQL HECTIC_INHERITANCE_SQL
'';
applyBundle = builtins.readFile self.lib.hectic.applyBundleScript;
migrator = hectic.writeShellApplication {
inherit shell bashOptions;
name = "migrator";
@@ -13,6 +28,8 @@ let
text = ''
${builtins.readFile hectic.helpers.posix-shell.log}
${hecticEnv}
${applyBundle}
${builtins.readFile ./migrator.sh}
'';
};

122
package/migrator/migrator.sh
View File

@@ -170,36 +170,20 @@ init() {
db_type=$(detect_db_type)
# INHERITS is PostgreSQL-only feature
[ ${INHERITS_LIST+x} ] && {
if [ "$db_type" != "postgresql" ]; then
log error "INHERITS is only supported for PostgreSQL"
exit 1
if [ "$db_type" = "postgresql" ]; then
if ! apply_hectic_bundle "$DB_URL"; then
log error "init failed: hectic bundle apply"
exit 13
fi
fi
oldIFS="$IFS"
IFS=','
check_inherits=
for table in $INHERITS_LIST; do
check_inherits="$(printf '%s\nSELECT 1 FROM %s LIMIT 1;' "$check_inherits" "$table")"
done
IFS="$oldIFS"
check_inherits=$(printf '%s\n' \
'BEGIN;' \
"$check_inherits" \
'COMMIT;')
if ! db_exec "$check_inherits"; then
log error "init failed: ${WHITE}one of inherits table does not exists: ${CYAN}$INHERITS_LIST"
exit 5
fi
}
if ! db_exec "$(init_sql)"; then
init_sql_extra="$(init_sql)"
if [ -n "$init_sql_extra" ]; then
if ! db_exec "$init_sql_extra"; then
log error "init failed"
exit 13
fi
fi
}
# error_handler_no_db_url()
@@ -209,86 +193,11 @@ error_handler_no_db_url() {
}
init_sql_postgresql() {
local sql inherits
inherits=
[ ${INHERITS_LIST+x} ] && inherits="$(printf 'INHERITS(%s)' "$INHERITS_LIST")"
sql="$(cat <<EOF
BEGIN;
DO \$\$
DECLARE
ver TEXT;
BEGIN
CREATE SCHEMA IF NOT EXISTS hectic;
-- NOTE(yukkop): check version table exists
IF EXISTS (
SELECT 1
FROM pg_class c
JOIN pg_namespace n ON n.oid = c.relnamespace
WHERE c.relname = 'version'
AND n.nspname = 'hectic'
AND c.relkind = 'r'
) THEN
SELECT v.version INTO ver FROM hectic.version v WHERE v.name = 'migrator';
IF ver IS NOT NULL AND ver != '$VERSION' THEN
RAISE EXCEPTION 'Incompatible migrator versions: % and $VERSION', ver;
END IF;
ELSE
CREATE TABLE hectic.version (
name TEXT PRIMARY KEY,
version TEXT NOT NULL,
installed_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
)$inherits;
END IF;
-- NOTE(yukkop): check migrator is registered in version table
IF NOT EXISTS (
SELECT 1 FROM hectic.version WHERE name = 'migrator'
) THEN
INSERT INTO hectic.version (name, version) VALUES ('migrator', '$VERSION');
END IF;
-- NOTE(yukkop): create migrator-specific objects if not yet present
IF NOT EXISTS (
SELECT 1
FROM pg_class c
JOIN pg_namespace n ON n.oid = c.relnamespace
WHERE c.relname = 'migration'
AND n.nspname = 'hectic'
AND c.relkind = 'r'
) THEN
CREATE DOMAIN hectic.migration_name AS TEXT CHECK (VALUE ~ '^[0-9]{14}-.*');
CREATE DOMAIN hectic.sha256 AS CHAR(64) CHECK (VALUE ~ '^[0-9a-f]{64}\$');
CREATE OR REPLACE FUNCTION hectic.sha256_lower() RETURNS trigger AS \$fn\$
BEGIN
NEW.hash = lower(NEW.hash);
RETURN NEW;
END;
\$fn\$ LANGUAGE plpgsql;
CREATE TABLE hectic.migration (
id SERIAL PRIMARY KEY,
name hectic.migration_name UNIQUE NOT NULL,
hash hectic.sha256 UNIQUE NOT NULL,
applied_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
)$inherits;
CREATE TRIGGER hectic_t_sha256_lower
BEFORE INSERT OR UPDATE ON hectic.migration
FOR EACH ROW EXECUTE FUNCTION hectic.sha256_lower();
END IF;
END;
\$\$;
COMMIT;
EOF
)"
printf '%s' "$sql"
# PostgreSQL init is delegated to apply_hectic_bundle (full hectic bundle
# applied as one unit on every `migrator init`). This function returns
# empty for the dry-run / db_exec path; bundle SQL paths are exposed via
# HECTIC_*_SQL env vars for inspection if needed.
printf ''
}
init_sql_sqlite() {
@@ -1102,7 +1011,7 @@ if ! [ "${AS_LIBRARY+x}" ]; then
shift 2
;;
--inherits)
INHERITS_LIST="${INHERITS_LIST+$INHERITS_LIST\"}$2"
log warn "--inherits is deprecated and ignored: hectic schema is auto-exempt from inheritance enforcement"
shift 2
;;
--*|-*) REMAINING_ARS="$REMAINING_ARS $(quote "$1")"; shift ;; # unknown global -> pass through
@@ -1110,7 +1019,6 @@ if ! [ "${AS_LIBRARY+x}" ]; then
esac
done
[ "${INHERITS_LIST+x}" ] && INHERITS_LIST="$(printf '%s' "$INHERITS_LIST" | sed -E 's/"/,/g; s/([^,]+)/"\1"/g')"
[ "${SUBCOMMAND+x}" ] || { log error "no subcommand specified. Use 'migrator help' for usage information."; exit 1; }