hectic-lab/util.nix
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(package): migrator: some migrate up works and init

Browse Source
This commit is contained in:
yukkop
2025-11-17 16:26:25 +00:00
parent 5b1f05589d
commit 9a7e7d677a
21 changed files with 276 additions and 89 deletions
Download Patch File Download Diff File Expand all files Collapse all files

105
package/migrator/migrator.sh
View File

@@ -17,6 +17,7 @@ if ! command -v psql >/dev/null; then
exit 127
fi
VERSION='0.0.1'
MIGRATION_DIR="${MIGRATION_DIR:-migration}"
quote() { printf "'%s'" "$(printf %s "$1" | sed "s/'/'\\\\''/g")"; }
REMAINING_ARS=
@@ -29,14 +30,14 @@ sha256sum() {
cksum --algorithm=sha256 --untagged "$file" | awk '{printf $1}'
}
INHERITS_LIST=
VARIABLE_LIST=
while [ $# -gt 0 ]; do
log debug "$1"
case $1 in
migrate|create|fetch|list|init)
[ "${SUBCOMMAND+x}" ] && { printf 'ambiguous subcommand, decide %s or %s\n' "$SUBCOMMAND" "$1"; exit 2; }
[ "${SUBCOMMAND+x}" ] && {
log error "ambiguous subcommand, decide ${WHITE}$SUBCOMMAND ${NC}or ${WHITE}$1";
exit 2;
}
SUBCOMMAND=$1
shift
;;
@@ -45,7 +46,7 @@ while [ $# -gt 0 ]; do
shift 2
;;
--inherits)
INHERITS_LIST="${INHERITS_LIST:+$INHERITS_LIST\"}$2"
INHERITS_LIST="${INHERITS_LIST+$INHERITS_LIST\"}$2"
shift 2
;;
--*|-*) REMAINING_ARS="$REMAINING_ARS $(quote "$1")"; shift ;; # unknown global -> pass through
@@ -53,7 +54,7 @@ while [ $# -gt 0 ]; do
esac
done
INHERITS_LIST="$(printf '%s' "$INHERITS_LIST" | sed -E 's/"/,/g; s/([^,]+)/"\1"/g')"
[ ${INHERITS_LIST+x} ] && INHERITS_LIST="$(printf '%s' "$INHERITS_LIST" | sed -E 's/"/,/g; s/([^,]+)/"\1"/g')"
# shellcheck disable=SC2120
init() {
@@ -68,7 +69,7 @@ init() {
shift 2
;;
--set|-v)
VARIABLE_LIST="${VARIABLE_LIST:+$VARIABLE_LIST }$2"
VARIABLE_LIST="${VARIABLE_LIST+$VARIABLE_LIST }$2"
shift 2
;;
--*|-*)
@@ -88,6 +89,7 @@ init() {
psql_args="$(form_psql_args)"
[ ${INHERITS_LIST+x} ] && {
oldIFS="$IFS"
IFS=','
check_inherits=
@@ -106,6 +108,7 @@ init() {
log error "init failed: ${WHITE}one of inherits table does not exists: ${CYAN}$INHERITS_LIST"
exit 5
fi
}
# shellcheck disable=SC2086
if ! psql $psql_args -c "$(init_sql)"; then
@@ -120,7 +123,6 @@ error_handler_no_db_url() {
}
init_sql() {
log debug "inherits: ${WHITE}${INHERITS_LIST}${NC}"
local sql
sql="$(printf '%s\n' \
"BEGIN;" \
@@ -138,7 +140,14 @@ init_sql() {
'END;' \
'$$ LANGUAGE plpgsql;' \
'' \
'CREATE SCHEMA IF NOT EXISTS hectic;' \
'CREATE TABLE IF NOT EXISTS hectic.version (' \
' name TEXT PRIMARY KEY,' \
' version TEXT NOT NULL,' \
' installed_at TIMESTAMPTZ NOT NULL DEFAULT NOW()' \
');' \
'' \
"INSERT INTO hectic.version (name, version) VALUES ('migrator', '$VERSION')" \
'' \
'CREATE TABLE IF NOT EXISTS hectic.migration (' \
' id SERIAL PRIMARY KEY,' \
' name hectic.migration_name UNIQUE NOT NULL,' \
@@ -146,7 +155,9 @@ init_sql() {
' applied_at TIMESTAMPTZ NOT NULL DEFAULT NOW()' \
')')"
sql="$(printf '%s INHERITS(%s);\n' "$sql" "$INHERITS_LIST")"
[ ${INHERITS_LIST+x} ] && sql="$(printf '%s INHERITS(%s)' "$sql" "$INHERITS_LIST")"
sql="$(printf '%s;\n' "$sql")"
printf '%s\n' \
"$sql" \
@@ -224,14 +235,22 @@ migrate_to() {
[ "${MIGRATION_NAME+x}" ] || { log error "no migration name specified"; exit 1; }
}
migration_list() {
find "$MIGRATION_DIR" -maxdepth 1 -type d -regextype posix-extended -regex '^.*/[0-9]{14}-.*$' -printf '%f\n' | sort
}
migrate() {
local fs_migrations db_migrations db_migration fs_migration psql_args var #target_migration
while [ $# -gt 0 ]; do
case $1 in
up|down|to)
[ -n "$MIGRATE_SUBCOMMAND" ] || (printf 'ambiguous migrate subcommand, decide %s or %s' "$MIGRATE_SUBCOMMAND" "$1"; exit 1)
[ "${MIGRATE_SUBCOMMAND+x}" ] && {
log error "ambiguous migrate subcommand, decide ${WHITE}$MIGRATE_SUBCOMMAND ${NC}or ${WHITE}$1";
exit 2
}
MIGRATE_SUBCOMMAND="$1"
shift
;;
--db-url|-u)
DB_URL="$2"
@@ -242,7 +261,7 @@ migrate() {
shift
;;
--set|-v)
VARIABLE_LIST="${VARIABLE_LIST:+$VARIABLE_LIST }$2"
VARIABLE_LIST="${VARIABLE_LIST+$VARIABLE_LIST }$2"
shift 2
;;
--*|-*) REMAINING_ARS="$REMAINING_ARS $(quote "$1")"; shift ;; # unknown global -> pass through
@@ -252,24 +271,24 @@ migrate() {
error_handler_no_db_url
[ -n "$FORCE" ] && {
[ "${FORCE+x}" ] && {
log error "migrate --force not implemented"
exit 1
}
init
fs_migrations=$(
find "$MIGRATION_DIR" -maxdepth 1 -type d -regex '^.*/[0-9]{15}-.*$' \
| sort \
| xargs -n1 basename
)
fs_migrations=$(migration_list)
db_migrations=$(
psql -Atqc "SELECT name FROM hectic.migration ORDER BY name ASC" \
psql "$DB_URL" --no-align --tuples-only --quiet \
--command "SELECT name FROM hectic.migration ORDER BY name ASC" \
| awk NF
)
db_mig_count=$(printf '%s' "$db_migrations" | wc)
log debug "$db_mig_count"
# Check if the DB migrations form a proper prefix of disk migrations
# (meaning all DB-applied migration filenames should appear in the same order at the start).
i=0
@@ -295,7 +314,7 @@ migrate() {
form_psql_args() {
psql_args="-d $DB_URL -v ON_ERROR_STOP=1"
for var in $VARIABLE_LIST; do
for var in ${VARIABLE_LIST:-}; do
psql_args="$psql_args -v $var"
done
}
@@ -307,14 +326,20 @@ migrate_inner() {
psql_args="$(form_psql_args)"
escaped_name=$(printf "%s" "$fs_migration" | sed "s/'/''/g")
escaped_path=$(printf "%s/%s/up.sql" "$MIGRATION_DIR" "$fs_migration" | sed "s/'/''/g")
direction=1
mig_direction=$([ "$direction" -gt 0 ] && printf 'up.sql' || printf 'down.sql')
escaped_name=$(printf '%s' "$fs_migration" | sed "s/'/''/g")
mig_path=$(printf '%s/%s/%s' "$MIGRATION_DIR" "$fs_migration" "$mig_direction")
escaped_path=$(printf '%s' "$mig_path" | sed "s/'/''/g")
log trace "mig name: $escaped_name; mig path: $escaped_path"
# shellcheck disable=SC2086
if ! psql $psql_args <<SQL
BEGIN;
\i '$escaped_path';
INSERT INTO hectic.migration (name) VALUES ('$escaped_name');
INSERT INTO hectic.migration (name, hash) VALUES ('$escaped_name', '$(sha256sum "$mig_path")');
COMMIT;
SQL
then
@@ -372,7 +397,39 @@ fetch() {
}
list() {
ls "$MIGRATION_DIR" -1
while [ $# -gt 0 ]; do
case $1 in
--raw|-r)
RAW=1
shift
;;
--*|-*)
log error "init argument $1 does not exists"
exit 9
;;
*)
log error "init subcommand $1 does not exists"
exit 9
;;
esac
done
[ "${RAW+x}" ] && {
migration_list
exit
}
migration_list | while read -r name; do
dir="./${MIGRATION_DIR}/${name}"
up="$dir/up.sql"
down="$dir/down.sql"
if [ ! -f "$up" ] || [ ! -f "$down" ]; then
echo "$name: missing $( [ ! -f "$up" ] && echo up.sql ) $( [ ! -f "$down" ] && echo down.sql )"
else
echo "$name"
fi
done
}
generate_word() {

27
test/package/migrator/test/arguments.sh Normal file
View File

@@ -0,0 +1,27 @@
log notice "test case: ${WHITE}error: ambiguous command"
set +e
migrator --inherits tablename --inherits 'table name' list migrate
error_code=$?
set -e
if [ "$error_code" = 0 ]; then
log error "test failed: ${WHITE}no error handler"
exit 1
elif [ "$error_code" != 2 ]; then
log error "test failed: ${WHITE}unexpected error code"
exit 1
fi
log notice "test case: ${WHITE}error: ambiguous migrate command"
set +e
migrator --inherits tablename --inherits 'table name' migrate to up
error_code=$?
set -e
if [ "$error_code" = 0 ]; then
log error "test failed: ${WHITE}no error handler"
exit 1
elif [ "$error_code" != 2 ]; then
log error "test failed: ${WHITE}unexpected error code"
exit 1
fi

76
test/package/migrator/test/init-migrator-with-inherits.sh Normal file
View File

@@ -0,0 +1,76 @@
#!/bin/dash
HECTIC_NAMESPACE=test-init-migrator
### CASE 1
log notice "test case: ${WHITE}dry run"
# NOTE: does not matter exist inherits tables or not, it must not connect to db
if ! migration_table_sql="$(migrator --inherits tablename --inherits 'table name' init --dry-run)"; then
log error "test failed: ${WHITE}error on migration table init dry run"
exit 1
fi
printf '%s' "$migration_table_sql" | grep -Eq 'INHERITS[[:space:]]*\([[:space:]]*"tablename"[[:space:]]*,[[:space:]]*"table name"[[:space:]]*\)' ||
{ log error "test failed: ${WHITE}not correct migration table inherits"; exit 1; }
### CASE 2
log notice "test case: ${WHITE}error: table inherit tables that not exists"
set +e
migrator --inherits tablename --inherits 'table name' init --db-url "$DATABASE_URL"
error_code=$?
set -e
if [ "$error_code" = 0 ]; then
log error "test failed: ${WHITE}no error handler"
exit 1
elif [ "$error_code" != 5 ]; then
log error "test failed: ${WHITE}unexpected error code"
exit 1
fi
### CASE 3
log notice "test case: ${WHITE}error: not provided --db-url"
set +e
migrator --inherits tablename --inherits 'table name' init
error_code=$?
set -e
if [ "$error_code" = 0 ]; then
log error "test failed: ${WHITE}no error handler"
exit 1
elif [ "$error_code" != 3 ]; then
log error "test failed: ${WHITE}unexpected error code"
exit 1
fi
### CASE 4
log notice "test case: ${WHITE}normal"
psql "$DATABASE_URL" -c 'CREATE TABLE "table name"(); CREATE TABLE tablename();'
if ! migrator --inherits tablename --inherits 'table name' init --db-url "$DATABASE_URL"; then
log error "test failed: ${WHITE}error on init sql"
exit 1
fi
if ! psql -v ON_ERROR_STOP=1 "$DATABASE_URL" -c 'SELECT * FROM hectic.migration'; then
log error "test failed: ${WHITE} tabe hectic.migration was not created"
exit 1
fi
### CASE 5
log notice "test case: ${WHITE}reinit (must just be ignored)"
if ! migrator --inherits tablename --inherits 'table name' init --db-url "$DATABASE_URL"; then
log error "test failed: ${WHITE}error on init sql"
exit 1
fi
if ! psql -v ON_ERROR_STOP=1 "$DATABASE_URL" -c 'SELECT * FROM hectic.migration'; then
log error "test failed: ${WHITE} tabe hectic.migration was not created"
exit 1
fi
log notice "test passed"

50
test/package/migrator/test/init-migrator.sh
View File

@@ -3,36 +3,17 @@
HECTIC_NAMESPACE=test-init-migrator
### CASE 1
log notice "test case: ${WHITE}table inherit tables that not exists"
log notice "test case: ${WHITE}dry run"
if ! migration_table_sql="$(migrator --inherits tablename --inherits 'table name' init --dry-run)"; then
if ! migration_table_sql="$(migrator init --dry-run)"; then
log error "test failed: ${WHITE}error on migration table init dry run"
exit 1
fi
printf '%s' "$migration_table_sql" | grep -Eq 'INHERITS[[:space:]]*\([[:space:]]*"tablename"[[:space:]]*,[[:space:]]*"table name"[[:space:]]*\)' ||
{ log error "test failed: ${WHITE}not correct migration table inherits"; exit 1; }
### CASE 2
log notice "test case: ${WHITE}error: table inherit tables that not exists"
set +e
migrator --inherits tablename --inherits 'table name' init --db-url "$DATABASE_URL"
error_code=$?
set -e
if [ "$error_code" = 0 ]; then
log error "test failed: ${WHITE}no error handler"
exit 1
elif [ "$error_code" != 5 ]; then
log error "test failed: ${WHITE}unexpected error code"
exit 1
fi
### CASE 3
log notice "test case: ${WHITE}error: not provided --db-url"
set +e
migrator --inherits tablename --inherits 'table name' init
migrator init
error_code=$?
set -e
@@ -45,14 +26,29 @@ elif [ "$error_code" != 3 ]; then
fi
### CASE 4
log notice "test case: ${WHITE}normal init"
log notice "test case: ${WHITE}normal"
psql "$DATABASE_URL" -c 'CREATE TABLE "table name"(); CREATE TABLE tablename();'
if ! migrator --inherits tablename --inherits 'table name' init --db-url "$DATABASE_URL"; then
if ! migrator --db-url "$DATABASE_URL" init; then
log error "test failed: ${WHITE}error on init sql"
exit 1
fi
psql -v ON_ERROR_STOP=1 "$DATABASE_URL" -c 'SELECT * FROM hectic.migration'
if ! psql -v ON_ERROR_STOP=1 "$DATABASE_URL" -c 'SELECT * FROM hectic.migration'; then
log error "test failed: ${WHITE} tabe hectic.migration was not created"
exit 1
fi
### CASE 5
log notice "test case: ${WHITE}reinit (must just be ignored)"
if ! migrator init --db-url "$DATABASE_URL"; then
log error "test failed: ${WHITE}error on init sql"
exit 1
fi
if ! psql -v ON_ERROR_STOP=1 "$DATABASE_URL" -c 'SELECT * FROM hectic.migration'; then
log error "test failed: ${WHITE} tabe hectic.migration was not created"
exit 1
fi
log notice "test passed"

22
test/package/migrator/test/migrate-up/run.sh
View File

@@ -1,10 +1,12 @@
#!/bin/dash
HECTIC_NAMESPACE=test-migration-list
psql "$DATABASE_URL" 'CREATE TABLE profile (
id INTEGER,
username TEXT
)'
#migrator migrate to 20251104192425-add-info-to-profile
##!/bin/dash
#
#HECTIC_NAMESPACE=test-migration-list
#
#psql "$DATABASE_URL" 'CREATE TABLE profile (
# id INTEGER,
# username TEXT
#)'
#
#migrator --db-url "$DATABASE_URL" migrate to 20251104192425-add-info-to-profile
#
#exit 1

1
test/package/migrator/test/migrations-list/migration/20251004192425-some-changes.sql
View File

@@ -1 +0,0 @@
-- nothing

0
test/package/migrator/test/migrations-list/migration/20251004192425-some-changes/placeholder Normal file
View File

1
test/package/migrator/test/migrations-list/migration/20251004292448-some-changes.sql
View File

@@ -1 +0,0 @@
-- nothing

0
test/package/migrator/test/migrations-list/migration/20251004292448-some-changes/down.sql Normal file
View File

0
test/package/migrator/test/migrations-list/migration/20251004292448-some-changes/up.sql Normal file
View File

1
test/package/migrator/test/migrations-list/migration/20251104172425-third-migration.sql
View File

@@ -1 +0,0 @@
-- nothing

0
test/package/migrator/test/migrations-list/migration/20251104172425-third-migration/up.sql Normal file
View File

1
test/package/migrator/test/migrations-list/migration/20251104192427-an-other-one.sql
View File

@@ -1 +0,0 @@
-- nothing

0
test/package/migrator/test/migrations-list/migration/20251104192427-an-other-one/up.sql Normal file
View File

1
test/package/migrator/test/migrations-list/migration/20251104292469-almoust-last.sql
View File

@@ -1 +0,0 @@
-- nothing

0
test/package/migrator/test/migrations-list/migration/20251104292469-almoust-last/down.sql Normal file
View File

0
test/package/migrator/test/migrations-list/migration/20251104292469-almoust-last/up.sql Normal file
View File

1
test/package/migrator/test/migrations-list/migration/20251204152446-very-last.sql
View File

@@ -1 +0,0 @@
-- nothing

0
test/package/migrator/test/migrations-list/migration/20251204152446-very-last/down.sql Normal file
View File

0
test/package/migrator/test/migrations-list/migration/20251204152446-very-last/up.sql Normal file
View File

41
test/package/migrator/test/migrations-list/run.sh
View File

@@ -3,14 +3,49 @@
HECTIC_NAMESPACE=test-migration-list
log notice "test case: ${WHITE}getting list of local migrations"
if ! list="$(migrator list)"; then
if ! result="$(migrator list --raw)"; then
log error "test failed: ${WHITE}error during execution"
exit 1
fi
ls
ls migration
printf '%s' "$result" > result
printf '20251004192425-some-changes
20251004292448-some-changes
20251104172425-third-migration
20251104192427-an-other-one
20251104292469-almoust-last
20251204152446-very-last' > expected
#printf 'result\n[\n%s\n]\n' "$(cat result)"
#printf 'expected\n[\n%s\n]\n' "$(cat expected)"
diff -q result expected || {
log error "test failed: ${WHITE}unexpected result"
exit 1
}
log notice "test case: ${WHITE}getting list of local migrations with info"
if ! result="$(migrator list)"; then
log error "test failed: ${WHITE}error during execution"
exit 1
fi
printf '%s' "$result" > result
printf '20251004192425-some-changes: missing up.sql down.sql
20251004292448-some-changes
20251104172425-third-migration: missing down.sql
20251104192427-an-other-one: missing down.sql
20251104292469-almoust-last
20251204152446-very-last' > expected
#printf 'result\n[\n%s\n]\n' "$(cat result)"
#printf 'expected\n[\n%s\n]\n' "$(cat expected)"
diff -q result expected || {
log error "test failed: ${WHITE}unexpected result"
exit 1
}
log notice "test passed"