feat(package): migrator: ! sqlite support

test/package/migrator/README.md

@@ -0,0 +1,164 @@
+# Migrator Test Suite
+
+This directory contains comprehensive tests for the database migration tool supporting both PostgreSQL and SQLite.
+
+## Test Structure
+
+```
+test/package/migrator/
+├── default.nix          # Nix test builder - auto-detects test type
+├── lauch.sh             # PostgreSQL test launcher
+├── lauch-sqlite.sh      # SQLite test launcher
+├── util.sh              # Shared test utilities
+└── test/                # Test cases
+    ├── <test-name>/     # PostgreSQL tests (default)
+    └── sqlite-<name>/   # SQLite tests (prefix with "sqlite-")
+```
+
+## Test Types
+
+### PostgreSQL Tests (Default)
+
+Any test directory or `.sh` file in `test/` will use PostgreSQL by default:
+- Automatic PostgreSQL setup (initdb, pg_ctl, createdb)
+- `DATABASE_URL` set to PostgreSQL connection string
+- Requires: `pkgs.postgresql`
+
+**Examples:**
+- `migrate-up-single/`
+- `migrate-down-multiple/`
+- `init-migrator.sh`
+
+### SQLite Tests
+
+Tests with names starting with `sqlite-` use SQLite:
+- Simple file-based database
+- `DATABASE_URL` set to `sqlite:///path/to/test.db`
+- Requires: `pkgs.sqlite`
+
+**Examples:**
+- `sqlite-basic/`
+- `sqlite-migration-test/`
+
+## Test Categories
+
+### Core Functionality
+- `init-migrator.sh` - Initialization
+- `init-migrator-with-inherits.sh` - PostgreSQL INHERITS feature
+- `migrate-up-single/` - Single step up migration
+- `migrate-up-multiple/` - Multiple step up migrations
+- `migrate-down-single/` - Single step down migration
+- `migrate-down-multiple/` - Multiple step down migrations
+- `migrate-to-forward/` - Migrate to specific version (forward)
+- `migrate-to-backward/` - Migrate to specific version (backward)
+- `migrate-already-at-target/` - Edge case: no-op migration
+
+### Existing Database Support
+- `migrate-existing-database/` - Add migrator to production DB
+- `migrate-existing-with-conflicts/` - Handle schema conflicts
+- `migrate-existing-data-migration/` - Transform existing data
+
+### SQLite Support
+- `sqlite-basic/` - Basic SQLite functionality
+
+### Helper Functions
+- `function-index-of.sh` - Test index_of helper
+- `function-migration-list.sh` - Test migration_list helper
+- `function-generate-word.sh` - Test word generator
+
+### Utilities
+- `create-migration.sh` - Test migration creation
+- `migrations-list/` - Test migration listing
+- `arguments.sh` - Test argument parsing
+
+## Creating New Tests
+
+### PostgreSQL Test
+
+```bash
+mkdir -p test/<test-name>/migration/<timestamp>-<name>
+cat > test/<test-name>/run.sh <<'EOF'
+#!/bin/dash
+HECTIC_NAMESPACE=test-my-test
+log notice "test case: ${WHITE}my test"
+
+# $DATABASE_URL is automatically set to PostgreSQL
+migrator --db-url "$DATABASE_URL" init
+# ... your test code ...
+
+log notice "test passed"
+EOF
+
+# Create up.sql and down.sql migration files
+```
+
+### SQLite Test
+
+Same as above, but prefix the directory name with `sqlite-`:
+
+```bash
+mkdir -p test/sqlite-<test-name>/migration/<timestamp>-<name>
+# ... rest is the same
+```
+
+## Running Tests
+
+Tests are built and run via Nix:
+
+```bash
+# Run all tests
+nix build .#checks.x86_64-linux
+
+# Run specific test
+nix build .#checks.x86_64-linux.migrator-test-<test-name>
+
+# Run SQLite tests
+nix build .#checks.x86_64-linux.migrator-test-sqlite-basic
+```
+
+## Test Isolation
+
+Each test runs in complete isolation:
+- **PostgreSQL**: Fresh PostgreSQL cluster per test
+- **SQLite**: Fresh database file per test
+- Clean working directory
+- Independent environment variables
+
+## Available Test Utilities
+
+From `util.sh`:
+- `columns(table)` - Get column names from table
+- `is_number(var)` - Check if variable is numeric
+
+From test environment:
+- `log <level> <message>` - Logging (trace, debug, info, notice, error)
+- `migrator` - The migrator binary under test
+- `$DATABASE_URL` - Database connection string (auto-configured)
+
+## Test Conventions
+
+1. **Naming**: Use descriptive names with hyphens
+2. **Logging**: Use `log` for output, not `echo`
+3. **Exit codes**: 
+   - 0 = success
+   - 1 = test failure
+   - Other = specific error conditions
+4. **Cleanup**: Tests are automatically cleaned up by Nix
+5. **Assertions**: Explicit checks with meaningful error messages
+
+## Database-Specific Notes
+
+### PostgreSQL
+- Full schema support (`hectic.migration`)
+- Domains with regex validation
+- Triggers and functions
+- INHERITS support
+- TIMESTAMPTZ support
+
+### SQLite
+- Simple table names (`hectic_migration`)
+- CHECK constraints instead of domains
+- No triggers needed
+- TEXT timestamps with datetime()
+- Table recreation for column removal (older SQLite versions)
+

test/package/migrator/default.nix

@@ -1,37 +1,41 @@
 { inputs, self, pkgs, system, ... }: let 
   lib = inputs.nixpkgs.lib;
 
-  # turn anything under ./test into a derivation that exposes $out/run.sh
-  mkTestDrv = name: type:
+  # turn anything under test directory into a derivation that exposes $out/run.sh
+  mkTestDrv = folder: name: type:
     if type == "directory" then
       pkgs.runCommand "test-${name}" {} ''
-        if ! [ -f ${./test + "/${name}" + /run.sh} ]; then
+        if ! [ -f ${"${folder}/${name}/run.sh"} ]; then
           echo no run.sh in test/${name}
           exit 1
         fi
 
         mkdir -p "$out"
-        cp -r ${./test + "/${name}"}/* "$out/"
+        cp -r ${"${folder}/${name}"}/* "$out/"
         chmod +x "$out/run.sh"
       ''
     else if lib.hasSuffix ".sh" name then
       pkgs.runCommand "test-${lib.removeSuffix ".sh" name}" {} ''
         mkdir -p "$out"
-        install -Dm755 ${./test + "/${name}"} "$out/run.sh"
+        install -Dm755 ${"${folder}/${name}"} "$out/run.sh"
       ''
     else
       null;
 
-  testDir = builtins.readDir ./test;
+  testDir = folder: builtins.readDir folder;
 
   # attrset: testName -> drv with run.sh
-  testDrvs =
+  testDrvs = folder:
     lib.mapAttrs' (n: v:
       lib.nameValuePair (lib.removeSuffix ".sh" n) v
     ) (lib.filterAttrs (_: v: v != null)
-      (lib.mapAttrs (n: t: mkTestDrv n t) testDir));
+      (lib.mapAttrs (n: t: mkTestDrv folder n t) (testDir folder)));
+
+  postgresqlTestDrvs = testDrvs ./test/postgresql;
+  sqliteTestDrvs     = testDrvs ./test/sqlite;
 
   migrator = self.packages.${system}.migrator;
+  
   mkPgTest = testName: testDrv: pkgs.runCommand "migrator-test-${testName}"
   {
     nativeBuildInputs = [ pkgs.coreutils pkgs.gnugrep pkgs.gnused ];
@@ -41,10 +45,26 @@
     test=${testDrv}
     export HECTIC_LOG=trace
     ${builtins.readFile ./util.sh}
-    ${builtins.readFile ./lauch.sh}
+    ${builtins.readFile ./lauch-postgresql.sh}
 
     # success marker for Nix
     # shellcheck disable=SC2154
     mkdir -p "$out"
   '';
-in lib.mapAttrs (name: drv: mkPgTest name drv) testDrvs
+
+  mkSqliteTest = testName: testDrv: pkgs.runCommand "migrator-test-${testName}"
+  {
+    nativeBuildInputs = [ pkgs.coreutils pkgs.gnugrep pkgs.gnused ];
+    buildInputs       = [ pkgs.which migrator pkgs.sqlite ];
+  } ''
+    ${builtins.readFile self.legacyPackages.${system}.helpers.posix-shell.log}
+    test=${testDrv}
+    export HECTIC_LOG=trace
+    ${builtins.readFile ./util.sh}
+    ${builtins.readFile ./lauch-sqlite.sh}
+
+    # success marker for Nix
+    # shellcheck disable=SC2154
+    mkdir -p "$out"
+  '';
+in (lib.mapAttrs (name: drv: mkPgTest name drv) postgresqlTestDrvs) // (lib.mapAttrs (name: drv: mkSqliteTest name drv) sqliteTestDrvs)

test/package/migrator/lauch-postgresql.sh

@@ -6,13 +6,12 @@
 HECTIC_NAMESPACE=test-laucher
 export HECTIC_LOG=trace
 
+# shellcheck disable=SC2154
 test_derivation="$(basename "$test")"
 test_name="${test_derivation#*-*-}"
 
 set -eu
 
-root_dir="$(dirname $0)"
-
 HECTIC_LOG=
 
 # save path to pg_ctl in case $PATH will change
@@ -65,8 +64,10 @@ log info "run test ${WHITE}${test_name}${NC}"
 mkdir './test'
 cp -r "$test"/* './test/'
 cd './test'
+# shellcheck disable=SC1091
 . './run.sh'
 
+# shellcheck disable=SC2034
 HECTIC_NAMESPACE=test-laucher
 
 log info "finish test pipeline"

test/package/migrator/lauch-sqlite.sh

@@ -0,0 +1,45 @@
+#!/bin/dash
+
+# $out  - nix derivation output
+# $test - test and assertion file
+
+HECTIC_NAMESPACE=test-laucher
+export HECTIC_LOG=trace
+
+# shellcheck disable=SC2154
+test_derivation="$(basename "$test")"
+test_name="${test_derivation#*-*-}"
+
+set -eu
+
+HECTIC_LOG=
+
+log info 'start test pipeline (SQLite)'
+
+# temp dirs
+wd="$PWD"
+db_file="$wd/test.db"
+
+# Set up SQLite database URL
+DATABASE_URL="sqlite://$db_file"
+export DATABASE_URL
+
+log info "using SQLite database: $db_file"
+log info "run test ${WHITE}${test_name}${NC}"
+
+# run test
+mkdir './test'
+cp -r "$test"/* './test/'
+cd './test'
+# shellcheck disable=SC1091
+. './run.sh'
+
+# shellcheck disable=SC2034
+HECTIC_NAMESPACE=test-laucher
+
+log info "finish test pipeline"
+
+# success marker for Nix
+# shellcheck disable=SC2154
+mkdir -p "$out"
+

test/package/migrator/test/postgresql/init-migrator-with-inherits.sh

@@ -2,18 +2,6 @@
 
 HECTIC_NAMESPACE=test-init-migrator
 
-### CASE 1
-log notice "test case: ${WHITE}dry run"
-# NOTE: does not matter exist inherits tables or not, it must not connect to db
-
-if ! migration_table_sql="$(migrator --inherits tablename --inherits 'table name' init --dry-run)"; then
-  log error "test failed: ${WHITE}error on migration table init dry run"
-  exit 1
-fi
-
-printf '%s' "$migration_table_sql" | grep -Eq 'INHERITS[[:space:]]*\([[:space:]]*"tablename"[[:space:]]*,[[:space:]]*"table name"[[:space:]]*\)' ||
-  { log error "test failed: ${WHITE}not correct migration table inherits"; exit 1; }
-
 ### CASE 2
 log notice "test case: ${WHITE}error: table inherit tables that not exists"
 

test/package/migrator/test/sqlite/sqlite-basic/migration/20250101000001-create-users/down.sql

@@ -0,0 +1,3 @@
+DROP TABLE users;
+
+

test/package/migrator/test/sqlite/sqlite-basic/migration/20250101000001-create-users/up.sql

@@ -0,0 +1,6 @@
+CREATE TABLE users (
+  id INTEGER PRIMARY KEY AUTOINCREMENT,
+  name TEXT NOT NULL
+);
+
+

test/package/migrator/test/sqlite/sqlite-basic/migration/20250101000002-add-email/down.sql

@@ -0,0 +1,12 @@
+-- SQLite doesn't support DROP COLUMN directly before 3.35.0
+-- We need to recreate the table
+CREATE TABLE users_new (
+  id INTEGER PRIMARY KEY AUTOINCREMENT,
+  name TEXT NOT NULL
+);
+
+INSERT INTO users_new (id, name) SELECT id, name FROM users;
+DROP TABLE users;
+ALTER TABLE users_new RENAME TO users;
+
+

test/package/migrator/test/sqlite/sqlite-basic/migration/20250101000002-add-email/up.sql

@@ -0,0 +1,3 @@
+ALTER TABLE users ADD COLUMN email TEXT;
+
+

test/package/migrator/test/sqlite/sqlite-basic/run.sh

@@ -0,0 +1,86 @@
+#!/bin/dash
+
+HECTIC_NAMESPACE=test-sqlite-basic
+
+log notice "test case: ${WHITE}SQLite basic migration"
+
+# Create SQLite database
+SQLITE_DB="$PWD/test.db"
+export DB_URL="sqlite://$SQLITE_DB"
+
+log info "using SQLite database: $SQLITE_DB"
+
+# Initialize migrator with SQLite
+if ! migrator --db-url "$DB_URL" init; then
+  log error "test failed: ${WHITE}init failed for SQLite"
+  exit 1
+fi
+
+# Verify tables were created
+if ! sqlite3 "$SQLITE_DB" "SELECT name FROM hectic_version WHERE name = 'migrator'" >/dev/null 2>&1; then
+  log error "test failed: ${WHITE}hectic_version table not created"
+  exit 1
+fi
+
+if ! sqlite3 "$SQLITE_DB" "SELECT COUNT(*) FROM hectic_migration" >/dev/null 2>&1; then
+  log error "test failed: ${WHITE}hectic_migration table not created"
+  exit 1
+fi
+
+log info "migrator tables created successfully"
+
+# Apply first migration
+if ! migrator --db-url "$DB_URL" migrate up; then
+  log error "test failed: ${WHITE}first migration failed"
+  exit 1
+fi
+
+# Verify migration was applied
+migration_count=$(sqlite3 "$SQLITE_DB" "SELECT COUNT(*) FROM hectic_migration")
+if [ "$migration_count" != "1" ]; then
+  log error "test failed: ${WHITE}expected 1 migration, got $migration_count"
+  exit 1
+fi
+
+# Verify table was created
+if ! sqlite3 "$SQLITE_DB" "SELECT COUNT(*) FROM users" >/dev/null 2>&1; then
+  log error "test failed: ${WHITE}users table not created"
+  exit 1
+fi
+
+log info "first migration applied successfully"
+
+# Apply second migration
+if ! migrator --db-url "$DB_URL" migrate up; then
+  log error "test failed: ${WHITE}second migration failed"
+  exit 1
+fi
+
+# Verify email column exists
+if ! sqlite3 "$SQLITE_DB" "SELECT email FROM users LIMIT 0" >/dev/null 2>&1; then
+  log error "test failed: ${WHITE}email column not added"
+  exit 1
+fi
+
+# Migrate down
+if ! migrator --db-url "$DB_URL" migrate down; then
+  log error "test failed: ${WHITE}migrate down failed"
+  exit 1
+fi
+
+# Verify only 1 migration remains
+migration_count=$(sqlite3 "$SQLITE_DB" "SELECT COUNT(*) FROM hectic_migration")
+if [ "$migration_count" != "1" ]; then
+  log error "test failed: ${WHITE}expected 1 migration after down, got $migration_count"
+  exit 1
+fi
+
+# Verify email column removed
+if sqlite3 "$SQLITE_DB" "SELECT email FROM users LIMIT 0" >/dev/null 2>&1; then
+  log error "test failed: ${WHITE}email column should be removed"
+  exit 1
+fi
+
+log notice "test passed: SQLite support works correctly"
+
+