feat: lfs

fix: world-of-sosal
feat: zomboid backups
2026-09-24 23:15:43 +00:00 · 2026-09-23 19:26:19 +00:00 · 2026-09-22 17:15:18 +00:00 · 2026-09-19 09:04:05 +00:00 · 2026-09-19 08:27:33 +00:00 · 2026-09-19 08:13:49 +00:00
214 changed files with 22596 additions and 842 deletions
@@ -0,0 +1,41 @@
+---
+# yamllint disable rule:line-length
+name: check Nix download buffer
+
+on: # yamllint disable-line rule:truthy
+  workflow_dispatch:
+
+concurrency:
+  group: check-nix-download-buffer
+  cancel-in-progress: true
+
+jobs:
+  check:
+    name: check Nix download buffer
+    runs-on: gross-nix-x86-perf
+    timeout-minutes: 20
+    env:
+      NIX_CONFIG: |
+        fallback = false
+        http2 = false
+        extra-substituters = https://cache.nixos.org https://cache.hectic-lab.com/hectic
+        trusted-public-keys = cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY= hectic:KMQsKow4SoA9K2vOJlOljmx7/Zpf91Yy+5qEtxDDCzA=
+    steps:
+      - name: Inspect Nix client
+        run: |
+          set -eu
+          nix --version
+          nix config show download-buffer-size
+          nix config show http2
+          nix config show fallback
+          nix config show substituters
+
+      - name: Ping Attic cache
+        run: |
+          set -eu
+          nix store ping --store https://cache.hectic-lab.com/hectic
+
+      - name: Download small cache path
+        run: |
+          set -eu
+          nix build --no-link --print-out-paths nixpkgs#hello
@@ -0,0 +1,86 @@
+---
+# yamllint disable rule:line-length
+name: deploy hectic-lab
+
+on: # yamllint disable-line rule:truthy
+  workflow_dispatch:
+
+concurrency:
+  group: deploy-hectic-lab
+  cancel-in-progress: false
+
+jobs:
+  deploy:
+    name: deploy hectic-lab with Attic cache
+    if: ${{ gitea.ref == 'refs/heads/master' }}
+    runs-on: nix
+    timeout-minutes: 435
+    env:
+      NIX_CONFIG: |
+        fallback = true
+        http2 = false
+        extra-substituters = https://cache.nixos.org https://cache.hectic-lab.com/hectic
+        trusted-public-keys = cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY= hectic:KMQsKow4SoA9K2vOJlOljmx7/Zpf91Yy+5qEtxDDCzA=
+    steps:
+      - name: Checkout dispatched revision
+        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683  # v4.2.2
+        with:
+          ref: ${{ gitea.sha }}
+          persist-credentials: false
+
+      - name: Show runner and cache configuration
+        run: |
+          set -eu
+          uname -a
+          nix --version
+          nix config show http2
+          nix config show fallback
+          nix config show download-buffer-size
+          nix config show substituters
+
+      - name: Configure SSH for hectic-lab
+        env:
+          HECTIC_LAB_SSH_PRIVATE_KEY: ${{ secrets.HECTIC_LAB_SSH_PRIVATE_KEY }}
+          HECTIC_LAB_SSH_KNOWN_HOSTS: ${{ secrets.HECTIC_LAB_SSH_KNOWN_HOSTS }}
+        run: |
+          set -eu
+          test -n "$HECTIC_LAB_SSH_PRIVATE_KEY"
+          test -n "$HECTIC_LAB_SSH_KNOWN_HOSTS"
+          ssh_home=$(mktemp -d)
+          trap 'rm -rf "$ssh_home"' EXIT
+          install -d -m 700 "$ssh_home/.ssh"
+          printf '%s\n' "$HECTIC_LAB_SSH_PRIVATE_KEY" > "$ssh_home/.ssh/id_ed25519"
+          printf '%s\n' "$HECTIC_LAB_SSH_KNOWN_HOSTS" > "$ssh_home/.ssh/known_hosts"
+          chmod 600 "$ssh_home/.ssh/id_ed25519" "$ssh_home/.ssh/known_hosts"
+          printf '%s\n' \
+            'Host hectic-lab' \
+            '  HostName 128.140.75.58' \
+            '  User root' \
+            "  IdentityFile $ssh_home/.ssh/id_ed25519" \
+            '  IdentitiesOnly yes' \
+            '  StrictHostKeyChecking yes' \
+            "  UserKnownHostsFile $ssh_home/.ssh/known_hosts" \
+            > "$ssh_home/.ssh/config"
+          chmod 600 "$ssh_home/.ssh/config"
+          printf '%s\n' "$ssh_home" > "$RUNNER_TEMP/hectic-lab-ssh-home"
+
+      - name: Deploy hectic-lab and save cache
+        env:
+          ATTIC_TOKEN: ${{ secrets.ATTIC_TOKEN }}
+          WITH_ATTIC_BUILD_TIMEOUT: "21600"
+          WITH_ATTIC_DRAIN_TIMEOUT: "3600"
+          WITH_ATTIC_UPLOAD_TIMEOUT: "600"
+          WITH_ATTIC_UPLOAD_RETRIES: "3"
+          WITH_ATTIC_BATCH_SIZE: "2"
+        run: |
+          set -eu
+          test -n "$ATTIC_TOKEN"
+          ssh_home=$(cat "$RUNNER_TEMP/hectic-lab-ssh-home")
+          trap 'rm -rf "$ssh_home"' EXIT
+          export HOME="$ssh_home"
+          export HECTIC_DEPLOY_SSH_CONFIG="$HOME/.ssh/config"
+          export NIX_SSHOPTS="-F $HOME/.ssh/config -o BatchMode=yes -o StrictHostKeyChecking=yes -o UserKnownHostsFile=$HOME/.ssh/known_hosts -o IdentitiesOnly=yes -i $HOME/.ssh/id_ed25519"
+          ssh -F "$HOME/.ssh/config" -o BatchMode=yes hectic-lab true
+          nix run --refresh '.#with-attic-cache' -- -- \
+            nix run --refresh '.#deploy' -- \
+              push -- --flake '.#hectic-lab|x86_64-linux' --target-host hectic-lab --use-remote-sudo
@@ -0,0 +1,113 @@
+---
+# yamllint disable rule:line-length
+name: deploy neuro
+
+on:  # yamllint disable-line rule:truthy
+  workflow_dispatch:
+    inputs:
+      upload_cache:
+        description: Upload build outputs to Attic
+        required: false
+        type: boolean
+        default: true
+      runner_label:
+        description: Runner label for deploy job
+        required: false
+        type: choice
+        default: nix
+        options:
+          - nix
+          - gross-nix-x86-perf
+          - gross-nix-x86-highmem
+
+concurrency:
+  group: deploy-neuro
+  cancel-in-progress: false
+
+jobs:
+  deploy:
+    name: deploy neuro from performance node
+    if: ${{ gitea.ref == 'refs/heads/master' }}
+    runs-on: ${{ gitea.event.inputs.runner_label || 'nix' }}
+    timeout-minutes: 435
+    env:
+      NIX_CONFIG: |
+        fallback = true
+        http2 = false
+        extra-substituters = https://cache.nixos.org https://cache.hectic-lab.com/hectic
+        trusted-public-keys = cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY= hectic:KMQsKow4SoA9K2vOJlOljmx7/Zpf91Yy+5qEtxDDCzA=
+    steps:
+      - name: Checkout dispatched revision
+        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683  # v4.2.2
+        with:
+          ref: ${{ gitea.sha }}
+          persist-credentials: false
+
+      - name: Show runner and cache configuration
+        run: |
+          set -eu
+          uname -a
+          nix --version
+          nix config show http2
+          nix config show fallback
+          nix config show substituters
+          nix config show trusted-public-keys
+
+      - name: Deploy neuro
+        env:
+          ATTIC_TOKEN: ${{ secrets.ATTIC_TOKEN }}
+          UPLOAD_CACHE: ${{ inputs.upload_cache }}
+          WITH_ATTIC_BUILD_TIMEOUT: "21600"
+          WITH_ATTIC_DRAIN_TIMEOUT: "3600"
+          WITH_ATTIC_UPLOAD_TIMEOUT: "600"
+          WITH_ATTIC_BATCH_SIZE: "4"
+          NEURO_SSH_PRIVATE_KEY: ${{ secrets.NEURO_SSH_PRIVATE_KEY }}
+          NEURO_SSH_KNOWN_HOSTS: ${{ secrets.NEURO_SSH_KNOWN_HOSTS }}
+        run: |
+          set -eu
+          test -n "$NEURO_SSH_PRIVATE_KEY"
+          test -n "$NEURO_SSH_KNOWN_HOSTS"
+          ssh_home=$(mktemp -d)
+          trap 'rm -rf "$ssh_home"' EXIT
+          install -d -m 700 "$ssh_home/.ssh"
+          printf '%s\n' "$NEURO_SSH_PRIVATE_KEY" > "$ssh_home/.ssh/id_ed25519"
+          printf '%s\n' "$NEURO_SSH_KNOWN_HOSTS" > "$ssh_home/.ssh/known_hosts"
+          chmod 600 "$ssh_home/.ssh/id_ed25519" "$ssh_home/.ssh/known_hosts"
+          printf '%s\n' \
+            'Host neuro' \
+            '  HostName 95.31.254.84' \
+            '  User root' \
+            '  Port 34457' \
+            "  IdentityFile $ssh_home/.ssh/id_ed25519" \
+            '  IdentitiesOnly yes' \
+            '  StrictHostKeyChecking yes' \
+            "  UserKnownHostsFile $ssh_home/.ssh/known_hosts" \
+            > "$ssh_home/.ssh/config"
+          chmod 600 "$ssh_home/.ssh/config"
+          export HOME="$ssh_home"
+          export HECTIC_DEPLOY_SSH_CONFIG="$HOME/.ssh/config"
+          export NIX_SSHOPTS="-F $HOME/.ssh/config -o BatchMode=yes -o StrictHostKeyChecking=yes -o UserKnownHostsFile=$HOME/.ssh/known_hosts -o IdentitiesOnly=yes -i $HOME/.ssh/id_ed25519"
+          ssh -F "$HOME/.ssh/config" -o BatchMode=yes -o ConnectTimeout=10 \
+            -o StrictHostKeyChecking=yes \
+            -o UserKnownHostsFile="$HOME/.ssh/known_hosts" \
+            -o IdentitiesOnly=yes \
+            -i "$HOME/.ssh/id_ed25519" \
+            neuro true
+          # Run deploy tool from checked-out Gitea revision; repository is not mirrored on GitHub.
+          case "$UPLOAD_CACHE" in
+            true)
+              nix run --refresh '.#with-attic-cache' -- -- \
+                nix run --refresh '.#deploy' -- \
+                  push -- --flake '.#neuro|x86_64-linux' --target-host neuro
+              ;;
+            false)
+              unset ATTIC_TOKEN
+              timeout --kill-after=60s 21600s \
+                nix run --refresh '.#deploy' -- \
+                  push -- --flake '.#neuro|x86_64-linux' --target-host neuro
+              ;;
+            *)
+              printf 'unsupported upload_cache value: %s\n' "$UPLOAD_CACHE" >&2
+              exit 64
+              ;;
+          esac
@@ -0,0 +1,43 @@
+name: runner nix smoke
+
+on:
+  workflow_dispatch:
+  push:
+    branches:
+      - master
+    paths:
+      - .gitea/workflows/runner-nix-smoke.yaml
+      - flake.lock
+      - flake.nix
+      - infra/gitea-runners/**
+      - lib/**
+      - legacy/**
+      - nixos/**
+      - package/**
+      - test/**
+
+jobs:
+  smoke:
+    name: nix label and flake smoke
+    runs-on: nix
+    env:
+      NIX_CONFIG: |
+        http2 = false
+        extra-substituters = https://cache.nixos.org https://cache.hectic-lab.com/hectic
+        extra-trusted-public-keys = hectic:KMQsKow4SoA9K2vOJlOljmx7/Zpf91Yy+5qEtxDDCzA=
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4.2.2
+
+      - name: Nix version and cache configuration
+        run: |
+          set -eu
+          nix --version
+          nix config show http2
+          nix config show substituters
+          nix config show trusted-public-keys
+
+      - name: Repository flake evaluation
+        run: |
+          set -eu
+          nix --option eval-cache false flake check --no-build --no-write-lock-file
@@ -0,0 +1,31 @@
+name: runner ubuntu smoke
+
+on:
+  workflow_dispatch:
+  push:
+    branches:
+      - master
+    paths:
+      - .gitea/workflows/runner-ubuntu-smoke.yaml
+      - infra/gitea-runners/**
+
+jobs:
+  smoke:
+    name: ubuntu-latest label smoke
+    runs-on: ubuntu-latest
+    steps:
+      - name: Basic runner information
+        run: |
+          set -eu
+          echo "hello from gitea runner"
+          uname -a
+
+      - name: Docker smoke when available
+        run: |
+          set -eu
+          if command -v docker >/dev/null 2>&1; then
+            docker version --format 'docker client={{.Client.Version}} server={{.Server.Version}}'
+            docker run --rm hello-world
+          else
+            echo "docker command not available; skipping Docker smoke"
+          fi
@@ -6,12 +6,19 @@ keys:
  - &bfs-server        age15yzgmsvl3ku2w863h6gw2vpmw37m9aruv6xrj4fue6n2jpm7pyuqk9xjmj
  - &bfs-pol-server    age1fpytf05sg9n6ywpwkmn09lhpfvgtud9h75h76jhxha475zpnasqq952rpu
  - &bfs-new-server    age17yx98qk9gzgcf2q6zhhp05p6mmtrkgz66dvyk9gqclypvlr8rersxjy5v7
-  - &neuro-server      age15yzgmsvl3ku2w863h6gw2vpmw37m9aruv6xrj4fue6n2jpm7pyuqk9xjmj
+  - &neuro-server      age1ak7heljpr0pjr4m0rcwxgn3sp0jjxw03lxyf33r8lcemqh2u2sgqx0aplq
  - &games-server      age15yzgmsvl3ku2w863h6gw2vpmw37m9aruv6xrj4fue6n2jpm7pyuqk9xjmj
  - &hectic-lab-server age13h8twnwvgxn04l5ywtru89a6psw5d0uckr2eghxsjp88a5augvsstq5ard
  - &umbriel-bfs       age1jxntjca8q2vxvf2jaal4xyvm2ae6sh62fhv897694kuzawfrk5asj00zdt

 creation_rules:
+  - path_regex: sus/neuro-minecraft.yaml$
+    key_groups:
+      - age:
+        - age1r25zdeqq8nac6dgca9en28r57ffyz9u9d8z5yc25gc8xqz747vaqmdtk0h
+        - age1ak7heljpr0pjr4m0rcwxgn3sp0jjxw03lxyf33r8lcemqh2u2sgqx0aplq
+        - age1ev53mzse6rg4ffwtcwtq4e93c7x7s4d0eyu89jrsahrke8r4yamsseu8h4
+
  - path_regex: sus/home.xray.yaml$
    key_groups:
      - age:
@@ -47,6 +54,15 @@ creation_rules:
        - *hectic-lab-server
        - *umbriel-bfs

+  - path_regex: sus/gitea-runners.yaml$
+    key_groups:
+      - age:
+        - *nrv
+        - *yukkop
+        - *yukkop-alt
+        - *hectic-lab-server
+        - *umbriel-bfs
+
  - path_regex: sus/matrix-cluster.yaml$
    key_groups:
      - age:
@@ -0,0 +1,21 @@
+{
+  flake,
+  self,
+  inputs,
+  system ? "aarch64-darwin",
+  ...
+}: let
+  name = builtins.baseNameOf ./.;
+in inputs.nix-darwin.lib.darwinSystem {
+  inherit system;
+  specialArgs = { inherit flake self inputs; };
+  modules = [
+    inputs.home-manager.darwinModules.home-manager
+    {
+      networking.hostName = name;
+      nixpkgs.hostPlatform = system;
+      nixpkgs.overlays = [ self.overlays.default ];
+    }
+    ./${name}.nix
+  ];
+}
@@ -0,0 +1,177 @@
+{
+  flake,
+  pkgs,
+  lib,
+  ...
+}: let
+  name = "yukkop";
+in {
+  system.primaryUser = name;
+  nix.settings.experimental-features = "nix-command flakes";
+
+  programs.zsh.enable = true;
+
+  services.openssh.enable = true;
+
+  users.users.${name} = {
+    home = "/Users/${name}";
+    openssh.authorizedKeys.keys = [
+      "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIJf9ljuqny71bZJokebK4Ybfml0MFMCkApS+tbMdBudp u0_a472@localhost"
+    ];
+  };
+
+  environment.systemPackages = with pkgs; [
+    aerospace
+    git
+    moreutils
+    neovim
+    tmux
+  ];
+
+  launchd.user.agents.aerospace = {
+    serviceConfig = {
+      ProgramArguments = [
+        "${pkgs.aerospace}/Applications/AeroSpace.app/Contents/MacOS/AeroSpace"
+      ];
+      RunAtLoad = true;
+      KeepAlive = true;
+      StandardOutPath = "/tmp/aerospace.out.log";
+      StandardErrorPath = "/tmp/aerospace.err.log";
+    };
+  };
+
+  home-manager.useGlobalPkgs = true;
+  home-manager.useUserPackages = true;
+  home-manager.backupFileExtension = "backup";
+  home-manager.sharedModules = [
+    (flake + "/home/module/program/tmux.nix")
+  ];
+  home-manager.users.${name} = {
+    home.stateVersion = "25.11";
+
+    home.packages = with pkgs; [
+      iproute2mac
+      jujutsu
+      ripgrep
+    ];
+
+    programs.git = {
+      enable = true;
+      lfs.enable = true;
+      settings = {
+        user.name = name;
+        user.email = "hectic.yukkop@gmail.com";
+        push.autoSetupRemote = true;
+        init.defaultBranch = "master";
+      };
+    };
+
+    programs.zsh = {
+      enable = true;
+      enableCompletion = true;
+      autosuggestion.enable = true;
+      syntaxHighlighting.enable = true;
+
+      history = {
+        size = 10000;
+        path = "$HOME/.zsh/.zsh_history";
+      };
+
+      shellAliases = {
+        drs = "darwin-rebuild switch --flake ~/pj/hearth#'yukkop|aarch64-darwin'";
+        nv = "nvim";
+        tmux = "tmux a";
+      };
+
+      initContent = ''
+        export PATH=/Users/yukkop/.opencode/bin:$PATH
+      '';
+    };
+
+    xdg.configFile."aerospace/aerospace.toml".text = ''
+      start-at-login = false
+
+      enable-normalization-flatten-containers = true
+      enable-normalization-opposite-orientation-for-nested-containers = true
+
+      default-root-container-layout = 'tiles'
+      default-root-container-orientation = 'auto'
+      accordion-padding = 30
+
+      on-focused-monitor-changed = ['move-mouse monitor-lazy-center']
+      automatically-unhide-macos-hidden-apps = false
+
+      [exec]
+        inherit-env-vars = true
+
+      [exec.env-vars]
+        PATH = '/run/current-system/sw/bin:/etc/profiles/per-user/yukkop/bin:/nix/var/nix/profiles/default/bin:/usr/local/bin:/usr/bin:/bin:/usr/sbin:/sbin:''${PATH}'
+
+      [gaps]
+        inner.horizontal = 8
+        inner.vertical = 8
+        outer.left = 8
+        outer.bottom = 8
+        outer.top = 8
+        outer.right = 8
+
+      [mode.main.binding]
+        alt-enter = 'exec-and-forget open -n /System/Applications/Utilities/Terminal.app'
+
+        alt-slash = 'layout tiles horizontal vertical'
+        alt-comma = 'layout accordion horizontal vertical'
+        alt-f = 'fullscreen'
+
+        alt-h = 'focus left'
+        alt-j = 'focus down'
+        alt-k = 'focus up'
+        alt-l = 'focus right'
+
+        alt-shift-h = 'move left'
+        alt-shift-j = 'move down'
+        alt-shift-k = 'move up'
+        alt-shift-l = 'move right'
+
+        alt-minus = 'resize smart -50'
+        alt-equal = 'resize smart +50'
+
+        alt-1 = 'workspace 1'
+        alt-2 = 'workspace 2'
+        alt-3 = 'workspace 3'
+        alt-4 = 'workspace 4'
+        alt-5 = 'workspace 5'
+        alt-6 = 'workspace 6'
+        alt-7 = 'workspace 7'
+        alt-8 = 'workspace 8'
+        alt-9 = 'workspace 9'
+
+        alt-shift-1 = 'move-node-to-workspace 1'
+        alt-shift-2 = 'move-node-to-workspace 2'
+        alt-shift-3 = 'move-node-to-workspace 3'
+        alt-shift-4 = 'move-node-to-workspace 4'
+        alt-shift-5 = 'move-node-to-workspace 5'
+        alt-shift-6 = 'move-node-to-workspace 6'
+        alt-shift-7 = 'move-node-to-workspace 7'
+        alt-shift-8 = 'move-node-to-workspace 8'
+        alt-shift-9 = 'move-node-to-workspace 9'
+
+        alt-tab = 'workspace-back-and-forth'
+        alt-shift-tab = 'move-workspace-to-monitor --wrap-around next'
+
+        alt-shift-semicolon = 'mode service'
+
+      [mode.service.binding]
+        esc = ['reload-config', 'mode main']
+        r = ['flatten-workspace-tree', 'mode main']
+        f = ['layout floating tiling', 'mode main']
+        b = ['balance-sizes', 'mode main']
+        backspace = ['close-all-windows-but-current', 'mode main']
+
+        alt-shift-h = ['join-with left', 'mode main']
+        alt-shift-j = ['join-with down', 'mode main']
+        alt-shift-k = ['join-with up', 'mode main']
+        alt-shift-l = ['join-with right', 'mode main']
+    '';
+  };
+  system.stateVersion = 6;
+}
@@ -8,6 +8,7 @@
  haskell    = import ./haskell.nix    { inherit self system pkgs; };
  neuro      = import ./neuro.nix      { inherit self system pkgs; };
  xmpp       = import ./xmpp.nix       { inherit self system pkgs; };
+  gitea-runners = import ./gitea-runners.nix { inherit pkgs; };
  default    = pkgs.mkShell {
    buildInputs =
      (with self.packages.${system}; [
@@ -0,0 +1,326 @@
+{ pkgs, ... }: let
+  opentofuUnstable = "github:NixOS/nixpkgs/nixos-unstable#opentofu";
+
+  tofu = pkgs.writeShellScriptBin "tofu" ''
+    exec ${pkgs.nix}/bin/nix run ${opentofuUnstable} -- "$@"
+  '';
+
+  packer = pkgs.stdenvNoCC.mkDerivation {
+    pname = "packer";
+    version = "1.16.0";
+    src = pkgs.fetchurl {
+      url = "https://releases.hashicorp.com/packer/1.16.0/packer_1.16.0_linux_amd64.zip";
+      hash = "sha256-XtzRSrWbU1BAxRLb7Nbsnvl2oACwc8Gdk+TEMclIWB4=";
+    };
+    nativeBuildInputs = [ pkgs.unzip ];
+    dontUnpack = true;
+    dontConfigure = true;
+    dontBuild = true;
+    installPhase = ''
+      install -d $out/bin
+      unzip -p $src packer > $out/bin/packer
+      chmod 755 $out/bin/packer
+    '';
+  };
+
+  buildMicroosSnapshots = pkgs.writeShellScriptBin "gitea-runners-build-microos-snapshots" ''
+    set -eu
+
+    : "''${HCLOUD_TOKEN:?HCLOUD_TOKEN is not set}"
+
+    architecture="''${1:-both}"
+    x86_location="''${GITEA_RUNNERS_X86_LOCATION:-nbg1}"
+    x86_server_type="''${GITEA_RUNNERS_X86_SERVER_TYPE:-cx23}"
+    case "$architecture" in
+      x86|arm|both) ;;
+      *)
+        printf 'usage: gitea-runners-build-microos-snapshots [x86|arm|both]\n' >&2
+        exit 2
+        ;;
+    esac
+
+    workdir="$(mktemp -d)"
+    trap 'rm -rf "$workdir"' EXIT HUP INT TERM
+    ssh_private_key_file="$workdir/packer-ssh-key"
+    ssh-keygen -q -t ed25519 -N "" -f "$ssh_private_key_file"
+    : "''${GITEA_RUNNERS_IMAGE_SSH_PUBLIC_KEY_FILE:?GITEA_RUNNERS_IMAGE_SSH_PUBLIC_KEY_FILE is not set}"
+    image_public_key_file="$GITEA_RUNNERS_IMAGE_SSH_PUBLIC_KEY_FILE"
+    test -r "$image_public_key_file"
+    packer_public_key_b64="$(base64 -w0 "$ssh_private_key_file.pub")"
+    image_public_key_b64="$(base64 -w0 "$image_public_key_file")"
+
+    curl -fsSL \
+      https://raw.githubusercontent.com/kube-hetzner/terraform-hcloud-kube-hetzner/master/scripts/create.sh \
+      -o "$workdir/create.sh"
+    chmod +x "$workdir/create.sh"
+
+    (
+      cd "$workdir"
+      folder_name="runner-images" \
+      folder_path="$workdir" \
+      create_snapshots=none \
+      HCLOUD_TOKEN="$HCLOUD_TOKEN" \
+      "$workdir/create.sh"
+    )
+
+    packer_dir="$workdir/runner-images/packer"
+    "$packer_dir/scripts/install-verified-packer-plugin-hcloud.sh"
+
+    # Packer may try to remove its remote script after the image-writing reboot,
+    # while SSH is already unavailable. Keep that cleanup from aborting builds.
+    sed -i \
+      '/inline[[:space:]]*=[[:space:]]*\[local\.write_x86_image\]/a\    skip_clean       = true' \
+      "$packer_dir/hcloud-microos-snapshots.pkr.hcl"
+    sed -i \
+      '/inline[[:space:]]*=[[:space:]]*\[local\.write_arm_image\]/a\    skip_clean       = true' \
+      "$packer_dir/hcloud-microos-snapshots.pkr.hcl"
+    sed -i \
+      '/inline[[:space:]]*=[[:space:]]*\[local\.install_packages\]/a\    start_retry_timeout = "15m"' \
+      "$packer_dir/hcloud-microos-snapshots.pkr.hcl"
+    sed -i \
+      '/ssh_username[[:space:]]*=[[:space:]]*"root"/a\  temporary_key_pair_type = "ed25519"' \
+      "$packer_dir/hcloud-microos-snapshots.pkr.hcl"
+    awk \
+      -v ssh_private_key_file="$ssh_private_key_file" \
+      '/token[[:space:]]*=[[:space:]]*var[.]hcloud_token/ {
+        print
+        print "  ssh_private_key_file = \"" ssh_private_key_file "\""
+        next
+      }
+      { print }' \
+      "$packer_dir/hcloud-microos-snapshots.pkr.hcl" \
+      > "$packer_dir/hcloud-microos-snapshots.pkr.hcl.tmp"
+    mv "$packer_dir/hcloud-microos-snapshots.pkr.hcl.tmp" \
+      "$packer_dir/hcloud-microos-snapshots.pkr.hcl"
+
+    cat > "$workdir/image-key-injection.txt" <<'EOF'
+   partprobe /dev/sda || true
+   udevadm settle
+   root_device=""
+   for candidate in /dev/sda[0-9]*; do
+     if [ "$(blkid -s TYPE -o value "$candidate" 2>/dev/null || true)" = btrfs ]; then
+       root_device="$candidate"
+       break
+     fi
+   done
+   test -n "$root_device"
+   mount -o subvolid=5,rw "$root_device" /mnt
+   default_subvolume="$(btrfs subvolume get-default /mnt | awk 'NR == 1 { print $2 }')"
+   umount /mnt
+   mount -o subvolid="$default_subvolume",rw "$root_device" /mnt
+   btrfs property set -ts /mnt ro false || true
+   install -d -m 0755 /mnt/nix /mnt/var/lib/gcr-nix
+   install -d -m 0700 /mnt/root/.ssh
+   printf '%s' '__IMAGE_PUBLIC_KEY_B64__' | base64 -d > /mnt/root/.ssh/gcr_authorized_keys
+   printf '\n%s' '__PACKER_PUBLIC_KEY_B64__' | base64 -d >> /mnt/root/.ssh/gcr_authorized_keys
+   chmod 0600 /mnt/root/.ssh/gcr_authorized_keys
+   install -d -m 0755 /mnt/etc/ssh/sshd_config.d
+   printf '%s\n' 'PermitRootLogin prohibit-password' 'PubkeyAuthentication yes' 'AuthorizedKeysFile .ssh/authorized_keys .ssh/gcr_authorized_keys' > /mnt/etc/ssh/sshd_config.d/99-gcr-root.conf
+   chmod 0644 /mnt/etc/ssh/sshd_config.d/99-gcr-root.conf
+   sync
+   umount /mnt
+EOF
+    sed -i "s|__PACKER_PUBLIC_KEY_B64__|$packer_public_key_b64|" \
+      "$workdir/image-key-injection.txt"
+    sed -i "s|__IMAGE_PUBLIC_KEY_B64__|$image_public_key_b64|" \
+      "$workdir/image-key-injection.txt"
+    awk -v inject_file="$workdir/image-key-injection.txt" \
+      '/rm -f \/root\/.ssh\/authorized_keys/ {
+         cleanup=1
+       }
+       cleanup && /sleep 1/ {
+         print
+         while ((getline line < inject_file) > 0) print line
+         close(inject_file)
+         cleanup=0
+         next
+       }
+      { print }' \
+      "$packer_dir/hcloud-microos-snapshots.pkr.hcl" \
+      > "$packer_dir/hcloud-microos-snapshots.pkr.hcl.tmp"
+    mv "$packer_dir/hcloud-microos-snapshots.pkr.hcl.tmp" \
+      "$packer_dir/hcloud-microos-snapshots.pkr.hcl"
+
+    cat > "$workdir/cloud-init-cleanup.txt" <<'EOF'
+  cloud-init clean --logs --machine-id --seed --configs all || true
+  rm -rf /run/cloud-init/* /var/lib/cloud/*
+EOF
+    awk -v cleanup_file="$workdir/cloud-init-cleanup.txt" \
+      '/# Cleanup some logs/ {
+        while ((getline line < cleanup_file) > 0) print "- [sh, -c, \"" line "\"]"
+        close(cleanup_file)
+      }
+      { print }' \
+      "$packer_dir/hcloud-microos-snapshots.pkr.hcl" \
+      > "$packer_dir/hcloud-microos-snapshots.pkr.hcl.tmp"
+    mv "$packer_dir/hcloud-microos-snapshots.pkr.hcl.tmp" \
+      "$packer_dir/hcloud-microos-snapshots.pkr.hcl"
+
+    x86_base='https://download.opensuse.org/tumbleweed/appliances'
+    x86_file='openSUSE-MicroOS.x86_64-ContainerHost-OpenStack-Cloud.qcow2'
+    x86_url="$x86_base/$x86_file"
+
+    arm_base='https://download.opensuse.org/ports/aarch64/tumbleweed/appliances'
+    arm_file='openSUSE-MicroOS.aarch64-ContainerHost-OpenStack-Cloud.qcow2'
+    arm_url="$arm_base/$arm_file"
+
+    x86_sha="$(curl -fsSL "$x86_url.sha256" | awk '{print $1; exit}')"
+    arm_sha="$(curl -fsSL "$arm_url.sha256" | awk '{print $1; exit}')"
+
+    test -n "$x86_sha"
+    test -n "$arm_sha"
+
+    printf 'x86 digest: %s\n' "$x86_sha"
+    printf 'arm digest: %s\n' "$arm_sha"
+
+    (
+      cd "$packer_dir"
+      packer init hcloud-microos-snapshots.pkr.hcl
+      case "$architecture" in
+        x86)
+          packer build \
+            -only=hcloud.microos-x86-snapshot \
+            -var 'selinux_package_to_install=k3s' \
+            -var "x86_location=$x86_location" \
+            -var "x86_server_type=$x86_server_type" \
+            -var "opensuse_microos_x86_expected_sha256=$x86_sha" \
+            -var "opensuse_microos_arm_expected_sha256=$arm_sha" \
+            hcloud-microos-snapshots.pkr.hcl
+          ;;
+        arm)
+          packer build \
+            -only=hcloud.microos-arm-snapshot \
+            -var 'selinux_package_to_install=k3s' \
+            -var "opensuse_microos_x86_expected_sha256=$x86_sha" \
+            -var "opensuse_microos_arm_expected_sha256=$arm_sha" \
+            hcloud-microos-snapshots.pkr.hcl
+          ;;
+        both)
+          packer build \
+            -var 'selinux_package_to_install=k3s' \
+            -var "x86_location=$x86_location" \
+            -var "x86_server_type=$x86_server_type" \
+            -var "opensuse_microos_x86_expected_sha256=$x86_sha" \
+            -var "opensuse_microos_arm_expected_sha256=$arm_sha" \
+            hcloud-microos-snapshots.pkr.hcl
+          ;;
+      esac
+    )
+  '';
+
+  giteaRunnersSetup = pkgs.writeShellScriptBin "gitea-runners-setup" /* sh */ ''
+    cat <<'EOF'
+Gitea runners setup checklist
+
+Tools available in this shell:
+  tofu, packer, gitea-runners-build-microos-snapshots [x86|arm|both], kubectl, kustomize,
+  kubeconform, sops, age, awscli2, hcloud,
+  tea, docker, skopeo, go-containerregistry, jq, yq-go, curl, git, openssh, nix
+
+Environment expected before real deploy/apply:
+  TF_VAR_hcloud_token
+  TF_VAR_ssh_public_key
+  TF_VAR_ssh_private_key
+  S3 backend credentials and endpoint access
+  a matching SOPS age identity for sus/gitea-runners.yaml
+  kubectl access to the target cluster
+  a valid Hetzner Nix image ID in the controller host configuration
+
+OpenTofu validation gate:
+  tofu version
+  tofu -chdir=infra/gitea-runners/opentofu validate
+
+Nix image build/publish gate:
+  nix build .#gitea-runner-nix-image
+  publish/import image, then set nixImageId / GCR_NIX_IMAGE_ID to its Hetzner ID
+
+SOPS token Secret creation gate:
+  kubectl apply -f infra/gitea-runners/k8s/namespace.yaml
+  umask 077
+  token_file=$(mktemp /tmp/gitea-runner-token.XXXXXX)
+  trap 'rm -f "$token_file"' EXIT
+  sops -d --extract '["gitea"]["hectic-lab"]["org-runner-registration-token"]' sus/gitea-runners.yaml > "$token_file"
+  kubectl -n gitea-runners create secret generic gitea-runner-token \
+    --from-file=token="$token_file" \
+    --dry-run=client \
+    -o yaml | kubectl -n gitea-runners apply -f -
+
+Cluster provision gate:
+  tofu -chdir=infra/gitea-runners/opentofu init
+  tofu -chdir=infra/gitea-runners/opentofu validate
+  tofu -chdir=infra/gitea-runners/opentofu plan -out=.sisyphus/evidence/task-12-deploy.plan
+  tofu -chdir=infra/gitea-runners/opentofu apply .sisyphus/evidence/task-12-deploy.plan
+  export KUBECONFIG="$(tofu -chdir=infra/gitea-runners/opentofu output -raw kubeconfig_path)"
+
+Kubernetes apply gate:
+  kubectl config current-context
+  kubectl get nodes -o wide
+  kubectl get sc
+  kubectl apply -k infra/gitea-runners/k8s
+
+Verification commands:
+  kubectl -n gitea-runners get statefulset gitea-runner
+  kubectl -n gitea-runners rollout status statefulset/gitea-runner --timeout=10m
+  kubectl -n gitea-runners get pods -l app.kubernetes.io/name=gitea-runner -o wide
+  kubectl -n gitea-runners get pvc -l app.kubernetes.io/name=gitea-runner -o wide
+  kubectl -n gitea-runners get events --sort-by=.lastTimestamp | tail -n 50
+  kubectl -n gitea-runners logs statefulset/gitea-runner -c runner --tail=200
+
+Main blockers and gates:
+  do not run tofu apply without all external inputs
+  do not apply the k8s overlay until the gitea-runner-token Secret exists
+  do not dispatch nix jobs until nixImageId / GCR_NIX_IMAGE_ID is valid
+  do not print, load, or require secrets on shell entry
+EOF
+  '';
+in pkgs.mkShell {
+  name = "gitea-runners";
+
+  buildInputs = [
+    tofu
+    packer
+    buildMicroosSnapshots
+    giteaRunnersSetup
+    pkgs.nix
+    pkgs.kubectl
+    pkgs.kustomize
+    pkgs.kubeconform
+    pkgs.sops
+    pkgs.age
+    pkgs.awscli2
+    pkgs.hcloud
+    pkgs.tea
+    pkgs.docker
+    pkgs.skopeo
+    pkgs.go-containerregistry
+    pkgs.jq
+    pkgs.yq-go
+    pkgs.curl
+    pkgs.git
+    pkgs.openssh
+    pkgs.unzip
+  ];
+
+  shellHook = ''
+    export GITEA_RUNNERS_ROOT="$PWD/infra/gitea-runners"
+    export GITEA_RUNNERS_TOFU_DIR="$GITEA_RUNNERS_ROOT/opentofu"
+    export GITEA_RUNNERS_K8S_DIR="$GITEA_RUNNERS_ROOT/k8s"
+    export GITEA_RUNNERS_IMAGE_DIR="$GITEA_RUNNERS_ROOT/image"
+    export GITEA_RUNNERS_NAMESPACE="gitea-runners"
+
+    alias cd-gitea-runners='cd "$GITEA_RUNNERS_ROOT"'
+    alias cd-gitea-runners-tofu='cd "$GITEA_RUNNERS_TOFU_DIR"'
+    alias cd-gitea-runners-k8s='cd "$GITEA_RUNNERS_K8S_DIR"'
+
+    echo ""
+    echo "=== Gitea runner setup DevShell ==="
+    echo ""
+    echo "Run gitea-runners-setup for the full setup checklist."
+    echo "Paths: "
+    echo "  root=$GITEA_RUNNERS_ROOT"
+    echo "  tofu=$GITEA_RUNNERS_TOFU_DIR"
+    echo "  k8s=$GITEA_RUNNERS_K8S_DIR"
+    echo "  image=$GITEA_RUNNERS_IMAGE_DIR"
+    echo ""
+  '';
+}
@@ -0,0 +1,32 @@
+# Documentation
+
+- [Using the `hectic` Attic Cache](./attic-cache.md)
+- [Minecraft incident log](./minecraft-incidents.md)
+- [Project Zomboid backups](./project-zomboid-backups.md)
+
+## Gitea runner labels
+
+Common labels for controller-managed on-demand runners:
+
+- `ubuntu-latest` — on-demand alias for `gross-x86`
+- `nix` — on-demand Nix alias with 480-minute TTL
+- `gross-x86` — x86 fallback chain `cx53` / `cx43` / `cx33`
+- `gross-arm` — ARM fallback chain `cax41` / `cax31` / `cax21`
+- `gross-x86-perf` — x86 performance chain `cx53` / `cpx62` / `cpx52`
+- `gross-mixed-econ` — mixed chain `cx53` / `cax41` / `cx43`
+- `gross-nix-x86` — `gross-x86` + Nix bootstrap
+- `gross-nix-arm` — `gross-arm` + Nix bootstrap
+- `gross-nix-x86-perf` — `gross-x86-perf` + Nix bootstrap
+- `gross-nix-x86-highmem` — CCX53-only Nix runner, 480-minute TTL
+- `gross-nix-mixed-econ` — `gross-mixed-econ` + Nix bootstrap
+
+Region order for fallback: `nbg1`, then `fsn1`, then `hel1`.
+
+The legacy Kubernetes persistent pool is disabled (`replicas: 0`) and has no
+registered labels. All listed labels are handled by the VM controller. After a
+successful job, a bootstrapped VM remains running and idle until its next hourly
+lifetime boundary, capped by label TTL. A queued job from same repository with
+same label reuses it without another VM creation or budget reservation.
+
+Operational details: `infra/gitea-runners/runbook.md` and
+`package/gitea-runner-controller/decide.sh`.
@@ -0,0 +1,330 @@
+# Using the `hectic` Attic Cache
+
+This document explains how to:
+
+1. pull build artifacts from the cache
+2. push new artifacts to the cache
+3. configure this flake to use the cache
+
+## Cache endpoints
+
+- API endpoint: `https://cache.hectic-lab.com`
+- Binary cache endpoint: `https://cache.hectic-lab.com/hectic`
+
+The `hectic` cache is:
+
+- public for reads
+- private for pushes
+
+## Requirements
+
+Use the Attic client package:
+
+```sh
+nix shell nixpkgs#attic-client
+```
+
+Or run commands directly with:
+
+```sh
+nix shell nixpkgs#attic-client -c <command>
+```
+
+## Read from the cache
+
+### Get the cache public key
+
+```sh
+nix shell nixpkgs#attic-client -c attic cache info hectic
+```
+
+Copy the `Public Key` value, which looks like:
+
+```text
+hectic:...
+```
+
+### Configure Nix to trust the cache
+
+Per-user: `~/.config/nix/nix.conf`
+
+```ini
+substituters = https://cache.nixos.org https://cache.hectic-lab.com/hectic
+trusted-public-keys = hectic:PASTE_PUBLIC_KEY_HERE
+```
+
+System-wide: `/etc/nix/nix.conf`
+
+```ini
+substituters = https://cache.nixos.org https://cache.hectic-lab.com/hectic
+trusted-public-keys = hectic:PASTE_PUBLIC_KEY_HERE
+```
+
+After that, normal Nix commands can download from the cache automatically:
+
+```sh
+nix build .#migrator
+nix develop
+nix flake check
+```
+
+## Use the cache from this flake
+
+You can also advertise the cache from `flake.nix`:
+
+```nix
+nixConfig = {
+  extra-substituters = [
+    "https://cache.nixos.org"
+    "https://cache.hectic-lab.com/hectic"
+  ];
+  extra-trusted-public-keys = [
+    "hectic:PASTE_PUBLIC_KEY_HERE"
+  ];
+};
+```
+
+Then users can run:
+
+```sh
+nix build --accept-flake-config .#migrator
+```
+
+## Log in for pushing
+
+Pushing requires an Attic token.
+
+```sh
+nix shell nixpkgs#attic-client -c attic login local https://cache.hectic-lab.com "<TOKEN>"
+```
+
+Example with `pass`:
+
+```sh
+nix shell nixpkgs#attic-client -c attic login local https://cache.hectic-lab.com "$(pass show atticd/hectic-lab/token)"
+```
+
+## Push build results
+
+### Push a package
+
+```sh
+nix build .#migrator
+nix shell nixpkgs#attic-client -c attic push local:hectic ./result
+```
+
+### Push a check
+
+```sh
+nix build .#checks.x86_64-linux.arguments
+nix shell nixpkgs#attic-client -c attic push local:hectic ./result
+```
+
+### Push a NixOS system build
+
+```sh
+nix build '.#nixosConfigurations."hectic-lab|x86_64-linux".config.system.build.toplevel'
+nix shell nixpkgs#attic-client -c attic push local:hectic ./result
+```
+
+## Recommended workflow
+
+### Local development
+
+Use the cache for reads only:
+
+```sh
+nix build .#migrator
+nix develop
+nix flake check
+```
+
+### CI / builder
+
+1. Build
+2. Push to Attic
+
+Example:
+
+```sh
+nix build .#migrator
+nix shell nixpkgs#attic-client -c attic push local:hectic ./result
+```
+
+## Useful commands
+
+### Show cache info
+
+```sh
+nix shell nixpkgs#attic-client -c attic cache info hectic
+```
+
+### Check login config
+
+```sh
+nix shell nixpkgs#attic-client -c attic cache info local:hectic
+```
+
+### Re-login with a new token
+
+```sh
+nix shell nixpkgs#attic-client -c attic login local https://cache.hectic-lab.com "<NEW_TOKEN>"
+```
+
+## Automatic uploads from trusted CI
+
+The `deploy-neuro` workflow uses `with-attic-cache` around its deployment command:
+
+```sh
+# ATTIC_TOKEN must be supplied through a secret, not committed or printed.
+nix run '.#with-attic-cache' -- -- nix build '.#my-package'
+```
+
+The wrapper installs a temporary Nix `post-build-hook`. Each successful local
+build queues all output paths, including build-only dependencies and multiple
+outputs. A separate worker uploads batches with `attic push --stdin --no-closure`
+and two concurrent uploads. Pending outputs have registered garbage-collection
+roots until uploaded. Substituted paths and the initial bootstrap of the wrapper
+itself are not uploaded; this avoids copying the public NixOS cache into Attic.
+
+The worker runs during the build and drains after success or failure. Uploads
+have bounded retries; exhausted uploads fail an otherwise successful command.
+If the build failed, its original exit status is preserved. Defaults are 30
+minutes for the wrapped command, 10 minutes for the final drain, and three
+120-second attempts per batch of up to 32 paths. These limits can be adjusted with
+`WITH_ATTIC_BUILD_TIMEOUT`, `WITH_ATTIC_DRAIN_TIMEOUT`,
+`WITH_ATTIC_UPLOAD_TIMEOUT`, `WITH_ATTIC_UPLOAD_RETRIES`, and
+`WITH_ATTIC_BATCH_SIZE` (positive integer seconds/counts without leading zeros).
+
+The heavier `deploy-neuro` workflow overrides these defaults: 6 hours for the
+build/deploy command, batches of at most 8 paths, and 600 seconds per upload attempt. The
+upload deadline covers the **whole batch**, not each individual path. Its final
+drain is bounded at 1 hour; the 435-minute job budget leaves 15 minutes for setup
+and cleanup. The `gross-nix-x86-perf` runner limit and Gitea's endless-task
+watchdog are 8 hours. VM hard lifetime starts at allocation and has no controller
+destruction grace. A prolonged cache outage can still exhaust the drain before
+every queued path is uploaded.
+
+The build timeout covers the entire wrapped command, not each derivation.
+Completed outputs can be reused from the cache, but an interrupted CUDA/Magma
+compilation does not produce a cacheable output or resume on the next ephemeral
+runner. Exit code 124 with `interrupted by the user` can therefore mean the
+wrapper deadline expired, not that someone manually cancelled the job.
+
+The workflow also sets `fallback = true` in `NIX_CONFIG`, inherited by nested
+Nix commands. If substitution fails, Nix can build the affected derivation from
+source instead of aborting solely because the cache is unavailable. Caches and
+signature checks remain enabled. Fallback cannot fix an unavailable upstream
+source or a genuine compilation error, and rebuilding can consume more time.
+
+Uploader logs report each attempt's batch size, whole-batch deadline and exit
+status, distinguish deadline expiration from other failures, and list paths in
+exhausted batches. The final summary counts queue records: acknowledged,
+unconfirmed after exhausted attempts, pending and in flight. These are not
+unique artifact counts: an unsuccessful batch may already have uploaded some
+paths, and a successful retry can reuse those cached results.
+
+This integration targets the root, single-user Nix environment on the ephemeral
+runner. It refuses to replace an existing post-build hook. SIGINT/SIGTERM stop
+the command and attempt a bounded drain; SIGKILL, VM destruction, or a hard
+runner timeout cannot guarantee uploads. A failed upload remains a visible CI
+failure, not a claim that the artifact was cached.
+
+### CI credentials and rollout
+
+- `ATTIC_TOKEN` is a Gitea repository secret for `hinterland/hearth`, passed only
+  to the deployment step. The workflow remains manual and restricted to `master`.
+- The token grants pull/push only for `hectic`, without deletion or cache
+  administration. The current token expires **2027-09-09**; rotate it before then.
+- The wrapper stores it in a private temporary `0600` file, references that file
+  from Attic configuration, and removes `ATTIC_TOKEN` from child environments.
+  Neither the hook nor `NIX_CONFIG` contains the token. Cleanup removes private
+  files after the worker stops.
+- Never expose this credential to untrusted PR workflows or bake it into runner
+  images. A writer to this cache can publish artifacts trusted by its consumers.
+- The `hectic` cache is public for reads. Build outputs must not contain secrets
+  or content that must remain private; the wrapper uploads every successful local
+  output, not just the final system.
+- Workflow/package changes must be published to `master` before dispatched runs
+  use them. Creating the secret alone does not enable uploads in an existing run.
+
+## Common issues
+
+### `flake 'nixpkgs' does not provide attribute 'attic'`
+
+Use:
+
+```sh
+nix shell nixpkgs#attic-client
+```
+
+Not:
+
+```sh
+nix shell nixpkgs#attic
+```
+
+### `HTTP 413 Payload Too Large`
+
+This means nginx rejected the upload body size. The server must allow large uploads on the Attic vhost.
+
+### Push succeeds for some paths but fails for others
+
+Usually means:
+
+- nginx body size limit
+- timeout/reverse proxy issue
+- bad token permissions
+
+On `hectic-lab`, the upload API has separate nginx locations for
+`/_api/v1/upload-path` and `/next/_api/v1/upload-path`. Requests stream to Attic
+without whole-body buffering, using HTTP/1.1 upstream and 600-second
+`proxy_send_timeout` and `proxy_read_timeout` values. These are inactivity
+timeouts, not an upload throughput guarantee. The CI wrapper still enforces its
+own whole-batch deadline. The legacy `/previous/` endpoint stays read-only.
+
+The host's Attic package also restricts its AWS SDK rustls connector to HTTP/1.1
+after observed S3 `REFUSED_STREAM` failures. This is a reproducible, host-scoped
+derived Cargo vendor tree; the pinned input tree and Cargo.lock are unchanged.
+TLS certificate verification remains enabled. Nix clients now force HTTP/1.1 for
+cache pulls because the cache endpoint has produced HTTP/2 framing errors; the
+Attic upload client separately uses HTTP/1.1 upstream. The pinned crate path
+makes upstream changes fail visibly during a future upgrade. This mitigates the
+observed transport error, not every possible Hetzner S3 timeout.
+
+### Cache pulls do not work
+
+Check:
+
+- `substituters`
+- `trusted-public-keys`
+- the exact public key from `attic cache info hectic`
+
+## Notes about retention and storage
+
+- The cache currently uses Hetzner Object Storage
+- If no `retention-period` is configured, cached objects do not expire automatically
+- This is good for long-lived reuse, but storage usage can grow over time
+
+## Summary
+
+### Read access
+
+```sh
+nix build .#migrator
+```
+
+after configuring:
+
+```ini
+substituters = https://cache.nixos.org https://cache.hectic-lab.com/hectic
+trusted-public-keys = hectic:PASTE_PUBLIC_KEY_HERE
+```
+
+### Push access
+
+```sh
+nix shell nixpkgs#attic-client -c attic login local https://cache.hectic-lab.com "<TOKEN>"
+nix build .#migrator
+nix shell nixpkgs#attic-client -c attic push local:hectic ./result
+```
@@ -0,0 +1,101 @@
+# Minecraft incident log
+
+This file records only observed evidence, actions, and verification results.
+An entity appearing in a stack trace is a trigger-path observation, not a
+proven root cause.
+
+## 2026-09-19 — WorldOfSosal crashes in Sable block-change handling
+
+### Impact
+
+- `minecraft-server-wowMineMap.service` terminates while a player is online.
+- Public Minecraft endpoint is `store.hectic-lab.com:25568`.
+- Server is intentionally stopped after the latest crash to prevent repeated
+  crash-save cycles while recovery is investigated.
+
+### Observed evidence
+
+All crash reports contain `sable@2.0.5` in
+`LevelAccelerator.getBlockState`, followed by
+`ArrayIndexOutOfBoundsException` where the requested section index exceeds
+the world section array length of `24`.
+
+| UTC timestamp | Crash report | Observed trigger path | Exception |
+| --- | --- | --- | --- |
+| 18:47:03 | `crash-2026-09-19_18.47.03-server.txt` | `EnderMan$EndermanTakeBlockGoal.tick` | index `38` / length `24` |
+| 18:52:17 | `crash-2026-09-19_18.52.17-server.txt` | `GlowSquid.aiStep` → `RedStoneOreBlock.stepOn` | index `33` / length `24` |
+| 19:14:46 | `crash-2026-09-19_19.14.46-server.txt` | `Skeleton.tick` → `RedStoneOreBlock.stepOn` | index `34` / length `24` |
+
+Evidence locations on `neuro`:
+
+```text
+/srv/minecraft/wowMineMap/crash-reports/
+/srv/minecraft/wowMineMap/logs/latest.log
+```
+
+### Actions performed
+
+| UTC timestamp | Action | Result |
+| --- | --- | --- |
+| 17:51 | Archived current world before recovery | Archive checksum recorded |
+| 18:08 | Set `randomTickSpeed=0` | Server started, but later crashed from an entity block change |
+| 18:48 | Set `mobGriefing=false` | Prevented Enderman block pickup only; later crashes still occurred |
+| 18:54 | Archived post-crash world | Archive checksum recorded |
+| 19:00 | Moved Boss offline player NBT from `(3299.067, 142.630, 8613.742)` to `(3296, 500, 8608)` in `crafting_azeroth:azeroth` | Only `Pos` and `Dimension` changed; later crash still occurred |
+| after 19:14 crash | Stopped `minecraft-server-wowMineMap.service` | Prevented further automatic crash/restart saves |
+
+### Recovery artifacts
+
+```text
+/srv/minecraft/backups/wowMineMap-before-sable-recovery-20260919T175139Z.tar.zst
+/srv/minecraft/backups/wowMineMap-after-sable-crashes-20260919T185445Z.tar.zst
+/srv/minecraft/wowMineMap/world/playerdata/1c189af5-2713-3fa6-bcc4-893dfadedfa4.dat.before-relocation
+```
+
+### Conclusions supported by evidence
+
+- Public proxy and reverse tunnel are not the failure point: server-list ping
+  succeeded before later in-world crashes.
+- The failure is not limited to Endermen, random ticks, or one player
+  position.
+- Sable's block-change callback is present in every captured crash.
+
+### Not established
+
+- Exact corrupt chunk, block, or mod data.
+- Whether world data is corrupt, Sable itself is defective, or another mod is
+  supplying incompatible world state.
+- Whether deleting any chunk, region, or Sable state would be safe.
+
+### External research
+
+No exact upstream match was found for Sable `2.0.5` on NeoForge `1.21.1` with
+`LevelAccelerator.getBlockState` and a requested section index of `33`, `34`,
+or `38` against a section array of length `24`.
+
+Related but non-identical upstream reports:
+
+- [Sable #776](https://github.com/ryanhcode/sable/issues/776) documents an
+  `ArrayIndexOutOfBoundsException` associated with unusual dimension height
+  bounds. This is relevant to section-coordinate handling, but is an older
+  version and different stack trace.
+- [Sable #1087](https://github.com/ryanhcode/sable/issues/1087) documents a
+  `LevelAccelerator.getBlockState` recursion during block-shape processing.
+  The failure type differs.
+- [Sable #820](https://github.com/ryanhcode/sable/issues/820) documents a
+  ticking-entity block-change crash. The reported downgrade to `1.1.3` helped
+  that distinct recursive-update failure; it is not evidence for this crash.
+- [Sable #1223](https://github.com/ryanhcode/sable/issues/1223) documents a
+  different `ArrayIndexOutOfBoundsException` in voxel-neighborhood handling.
+  Its suggested Lithium setting only reduced crashes for some reporters and is
+  not a verified mitigation here.
+
+Sable `2.0.4` and `2.0.5` release notes mention other block or contraption
+crash fixes, but not this exception. No version upgrade or downgrade is
+currently evidence-backed as a production fix.
+
+### Next recovery step
+
+Use a disposable full-world copy to test a supported Sable/physics integration
+mitigation. Do not restart production, delete region files, or overwrite a
+backup until that test gives reproducible evidence.
@@ -0,0 +1,231 @@
+# WorldOfSosal: Prism automatic updates
+
+The published client entry points are:
+- https://store.bfs.band/minecraft/ (BFS / Element host)
+- https://store.hectic-lab.com/minecraft/world-of-sosal/ (hectic-lab)
+
+Each site provides its own Prism ZIP with that site's update URL and matching
+server address. Both installs use the same Minecraft world and modpack release.
+
+Players import `WorldOfSosal-Prism.zip` into Prism once and approve its pre-launch
+command. Before each launch, packwiz-installer reconciles the client with the
+published pack: it adds, replaces, and removes managed files, checking hashes.
+`options.txt` is seeded once and preserved. Pack configuration files are managed
+and can be replaced. Upstream mods do not update independently of your release.
+Minecraft 1.21.1, NeoForge 21.1.250, Java 21; the instance reserves up to 8 GiB.
+
+The original `.mrpack` alone does not provide this automatic update mechanism.
+Official workflow: https://packwiz.infra.link/tutorials/installing/packwiz-installer/
+
+## Publishing a tested update
+
+Keep the authoritative `.mrpack` in Storage Box at
+`minecraft/pack/WorldOfSosal.mrpack`. For a server update, replace that archive,
+set its new SHA-256 in `nixos/system/neuro/minecraft/world-of-sosal.nix`,
+and rebuild/switch neuro before publishing the corresponding client export. The server importer and the
+client export must consume the same archive; publishing only the client can make
+it incompatible with the running server.
+
+```sh
+# Test the client and deploy the matching server release first.
+python3 script/publish-prism-mirrors.py WorldOfSosal.mrpack
+```
+
+The mirror publisher creates temporary build directories and sets each server
+address and update URL automatically. The builder downloads a SHA-256-pinned bootstrap from the
+packwiz project's release, or accepts it via `--bootstrap /path/to/file.jar`.
+External mods retain their original URLs and SHA-512 checksums. Embedded mods and
+configuration are hosted with the release. Both required and optional client mods
+are included, matching the current server importer's optional-mod behavior.
+
+Publishing uploads an immutable directory, checks it if it already exists, and
+atomically switches `current`. Previous directories remain available for rollback.
+Do not remove a release while clients may still be reading it. Hash checks cause
+an overlapping update to fail safely rather than silently accept mixed contents;
+retry the launch if a publication overlapped a download.
+
+The files live under `/var/www/store/minecraft/world-of-sosal` on `hectic-lab`,
+served by the existing `store.hectic-lab.com` nginx virtual host. No nginx reload
+is needed for pack updates. Keep `current/pack.toml` as the stable client URL.
+The index must be alongside pack.toml: putting a release prefix in `[index].file`
+also prefixes client installation paths with that directory in packwiz-installer.
+
+If Minecraft/NeoForge versions change, update and test both the server pin and
+client pack. packwiz-installer 0.5.14 understands NeoForge components in Prism's
+`mmc-pack.json`; a launcher restart/relaunch may be necessary after changing them.
+
+## Verification on 2026-09-18
+
+- Source archive SHA-256:
+  `f8c18acb9208e4592725632ae50dab4f9c308483b34fd43a6507c74fdbf8169f`.
+- Public HTTPS installation into a clean Prism-format instance passed: all 141
+  client mods and all overrides match the original archive. A second launch
+  performed no downloads and preserved personal options.
+- Direct probes of neuro public ports 25565, 25567, and 25568 timed out;
+  the configured relay now provides the public entry point.
+- Live WoW server reached `Done` with all 135 server mod SHA-512 hashes
+  matching the same archive used for the Prism client.
+- Public `store.hectic-lab.com:25568` status/ping succeeded (about 111 ms);
+  a login handshake reached the online authentication encryption request.
+  An authenticated Windows Prism session was subsequently verified on 2026-09-19 (see below).
+- Server and tunnel are enabled at boot; relay and both NixOS configurations
+  are deployed. No failed systemd units remain on neuro.
+- Loader package `neoforge-1.21.1-21.1.250` built successfully in Nix.
+- Automatic updater add/remove/config-update and options-preservation behavior
+  tested with an actual packwiz-installer run against a controlled update fixture.
+
+## WoW server and public entry point
+
+The WoW map and WorldOfSosal mods share the `wowMineMap` server on neuro,
+listening on 25567. There is no separate WorldOfSosal world/server on 25568.
+The client pack and server both pin Minecraft 1.21.1 / NeoForge 21.1.250.
+Map import runs before mod import, and both finish before Minecraft starts.
+
+The public entry point is `store.hectic-lab.com:25568`:
+
+```
+Prism -> hectic-lab:25568 -> loopback:25577 -> SSH tunnel -> neuro:25567
+```
+
+`minecraft-wow-proxy.socket` and its socket-proxyd service run on hectic-lab.
+`minecraft-wow-tunnel.service` on neuro establishes a reverse SSH forward and
+reconnects after failures. A dedicated SSH identity may listen only on
+127.0.0.1:25577 at the relay; it has no interactive shell or other forwarding.
+Both services and firewall rules are in Nix and start on boot. The SSH client
+uses an explicit AES-CTR / HMAC-SHA256-ETM / curve25519 transport profile with
+IPQoS=none, tested on the neuro-to-lab route. The default profile stalled after
+the handshake on this route. Both ends check peer liveness so stale listeners
+are eventually released. Minecraft initially used `online-mode=true`. It now uses offline mode at the
+owner's request; see the RCON and authentication section below.
+
+For a temporary direct local tunnel, use:
+
+```sh
+ssh -NTL 0.0.0.0:25568:127.0.0.1:25567 \
+  -o ExitOnForwardFailure=yes \
+  -o ServerAliveInterval=30 -o ServerAliveCountMax=3 neuro
+```
+
+That command exposes the local 25568 listener on all interfaces, as requested.
+Use 127.0.0.1 instead of the first 0.0.0.0 if only this computer should use it.
+
+Credentials are encrypted in `sus/neuro-minecraft.yaml` with the actual neuro
+host identity and owner keys. The existing `sus/neuro.yaml` is unchanged.
+The source WoW archive remains untouched in Storage Box. Import is idempotent:
+an existing world with level.dat is preserved. Never delete the world to update
+mods; publish/deploy a matching modpack release instead.
+
+Useful checks:
+
+```sh
+ssh neuro systemctl status minecraft-world-import-wowMineMap \
+  minecraft-modpack-import-worldOfSosal minecraft-server-wowMineMap \
+  minecraft-wow-tunnel --no-pager
+ssh hectic-lab systemctl status minecraft-wow-proxy.socket --no-pager
+ssh neuro journalctl -u minecraft-server-wowMineMap -n 80 --no-pager
+```
+
+The initial isolated server compatibility test reached `Done` and answered the
+Minecraft status/ping protocol. Its logs also contain nonfatal recipe and class
+function errors from the supplied modpack; successful startup does not imply that
+every recipe or RPG class feature works correctly.
+
+The imported map metadata is `wow mine`, DataVersion 3953 (Minecraft 1.21),
+spawn 0 / 68 / -32; extracted size is approximately 11.7 GiB. The archive
+SHA-256 was verified before extraction.
+
+## Windows Prism GUI verification on 2026-09-19
+
+- Downloaded the published ZIP through the browser and imported it in Prism 8.4.
+- Fixed the generated instance.cfg: ConfigVersion=1.2 is required. Without it,
+  Prism selects its legacy INI parser and corrupts the quoted pre-launch command.
+  The corrected ZIP is published at the same URL. Previously imported copies
+  need the command corrected in Settings / Custom commands, or a fresh import.
+- Used Java 21.0.4; the first packwiz download hit two transient timeouts.
+  Cancelled the incomplete launch and retried successfully. All 141 downloaded
+  client mod hashes match the original mrpack. NeoForge reports 202 mods when
+  bundled/internal mod components are included.
+- Joined store.hectic-lab.com:25568 in the actual Minecraft GUI. The server
+  confirmed the authenticated join, and the client reached the Origins selection
+  screen. No character origin was selected during testing.
+- Tested a separate copy of the pack manifest with an inert config text file:
+  launching from Prism added it; restoring the production manifest and launching
+  again automatically deleted it. Existing files were reused from cache, and
+  options.txt retained its checksum. The production pack contents were unchanged.
+- Restored the instance's regular current/pack.toml update URL.
+
+## Independent BFS entry point (2026-09-19)
+
+- Server: `wow.bfs.band`; downloads: https://store.bfs.band/minecraft/.
+- BFS is `bfs.poland.xray` (91.198.166.181), the host of Element.
+- `minecraft-wow-tunnel-bfs` connects neuro directly to BFS. The BFS path does
+  not transit hectic-lab; both tunnels have independent reconnecting services.
+- Shared proxy implementation: `nixos/module/generic/minecraft-public-relay.nix`.
+  Host settings remain in `minecraft-wow-proxy.nix` (hectic-lab) and
+  `minecraft-wow.nix` (BFS). A dedicated HTTPS virtual host serves `store.bfs.band`. The legacy
+  `bfs.band/minecraft/` URLs remain available for already imported instances.
+- Downloaded BFS ZIP seeds `wow.bfs.band` and uses the stable manifest
+  `https://store.bfs.band/minecraft/world-of-sosal/current/pack.toml`. It does not
+  redirect installation metadata to hectic-lab. Upstream mod and Java/loader
+  downloads still use their original providers (e.g. Modrinth, GitHub, Mojang).
+- Existing hectic-lab instances can be migrated without reinstalling mods:
+  in Edit / Settings / Custom commands, replace only the manifest URL in
+  Pre-launch command with the BFS URL above. Change the multiplayer server
+  address to wow.bfs.band. New users should import the ZIP from BFS.
+- `script/publish-prism-mirrors.py` builds host-specific ZIPs from one archive
+  and publishes both mirrors. It checks that the running neuro server's cached
+  archive has the same SHA-256. Each host's switch is atomic; publication across
+  two hosts is sequential, so rerun the command if it exits unsuccessfully.
+- Both configurations were deployed; public Minecraft status/ping succeeds
+  on BFS (~125 ms), HTTPS serves the pack, and Element/Matrix HTTP checks pass.
+
+Clean installation through the BFS manifest passed: all 141 client mods and
+all overrides match the source archive. A second updater run performed no
+downloads and preserved options.txt. The public BFS login protocol reached
+online authentication; the earlier full GUI login used hectic-lab.
+
+## BFS DNS and dedicated download site (2026-09-19)
+
+Porkbun DNS, TTL 600:
+
+| Type | Name | Value |
+| --- | --- | --- |
+| A | store.bfs.band | 91.198.166.181 |
+| A | wow.bfs.band | 91.198.166.181 |
+| SRV | _minecraft._tcp.wow.bfs.band | 0 0 25568 wow.bfs.band |
+
+Players enter `wow.bfs.band` without a port in Minecraft Java. In Porkbun,
+SRV Priority is `0`, and Target is `0 25568 wow.bfs.band` (weight, port, host).
+The root download URL https://store.bfs.band/ redirects to the WorldOfSosal page.
+The NixOS virtual host obtains and renews its HTTPS certificate automatically.
+The publication script now seeds this update URL and the port-free game address.
+Existing BFS instances retain working legacy update URLs; switching their
+pre-launch manifest to the new store host is optional. Root bfs.band remains
+the existing Element entry point.
+
+## RCON and authentication (2026-09-19)
+
+The WoW server now has `online-mode=false`. Account authentication is disabled;
+player names can be impersonated, and offline UUIDs differ from online UUIDs.
+Existing inventory/permissions may require a separate UUID migration.
+
+RCON listens on TCP 25575 on neuro; its port is not opened in the firewall or
+forwarded through the public Minecraft relays. The server-specific automatic
+firewall is disabled and only game port 25567 is explicitly permitted.
+A random password is stored in SOPS as `minecraft/rcon-password`, injected into
+server.properties at startup with mode 0600, and is absent from the Nix store.
+
+Start a local-only SSH tunnel and leave it running:
+
+```sh
+ssh -NT -L 127.0.0.1:25575:127.0.0.1:25575 -o ExitOnForwardFailure=yes -o ServerAliveInterval=30 -o ServerAliveCountMax=3 neuro
+```
+
+Retrieve the password in another terminal (do not paste it into logs):
+
+```sh
+ssh neuro cat /run/secrets/minecraft/rcon-password
+```
+
+Configure the RCON client with host `127.0.0.1`, port `25575`, and that password.
+There is no RCON username. These changes apply to wowMineMap only.
@@ -0,0 +1,96 @@
+# Project Zomboid backups
+
+`hectic.services."project-zomboid".backup` creates local backups without stopping
+or pausing the server. The default schedule is every 30 minutes. Each run:
+
+1. sends the local RCON `save` command and waits for the configured save grace
+   period;
+2. rsyncs `Zomboid/Saves/Multiplayer/<serverName>` and non-secret server
+   settings (`SandboxVars`, spawn-points, and spawn-regions) from
+   `Zomboid/Server` into a private staging tree;
+3. waits five seconds and repeats the rsync to narrow the live-write window;
+4. publishes a timestamped `tar.zst` archive; and
+5. deletes local archives older than `backup.retentionDays`.
+
+The service lock prevents overlapping runs. Missing save or server-config paths
+skip the run through systemd `ConditionPathExists` checks.
+
+## Consistency and secrets
+
+This is a best-effort backup. It does not stop Project Zomboid and does not use
+an atomic filesystem snapshot. The RCON save command flushes the world before
+copying, and the second rsync narrows the remaining live-write window, but
+neither makes the filesystem copy an atomic snapshot.
+
+Archives do not include the generated server INI, `admin-password`,
+host-generated password files, or the S3 credentials file. The server INI is
+generated again during service startup; provision secret-backed values separately
+after a restore.
+
+## hectic-lab
+
+hectic-lab runs the timer every 30 minutes and keeps local archives for 14 days:
+
+```text
+/var/lib/project-zomboid/backups/archive/
+```
+
+Check it with:
+
+```sh
+systemctl list-timers project-zomboid-backup.timer
+systemctl status project-zomboid-backup.service
+journalctl -u project-zomboid-backup.service
+```
+
+RCON is enabled on localhost port `27015`; the firewall does not expose this
+port. The password is generated at
+`/var/lib/project-zomboid/rcon-password` with mode `0600`. The server also uses
+`SaveWorldEveryMinutes=15` as a periodic persistence fallback.
+
+## Optional S3 upload
+
+S3 upload is disabled by default. Enabling it requires `bucket`, `endpoint`,
+`region`, and an absolute runtime `credentialsFile` outside `/nix/store`. The
+endpoint must use HTTPS. systemd reads the environment file without executing
+it; this host keeps it owned by `project-zomboid` with mode `0400`:
+
+```sh
+AWS_ACCESS_KEY_ID=...
+AWS_SECRET_ACCESS_KEY=...
+```
+
+Set `backup.s3.prefix` to choose the object-key prefix and
+`backup.s3.remoteRetentionDays` to prune old archives from that prefix. Remote
+deletion runs only after a successful upload and only matches this server's
+archive name prefix. Configure bucket lifecycle expiration/versioning too when
+available; it remains the stronger recovery and cleanup control.
+
+## Restore
+
+Restoring must be done while the server is stopped so it cannot modify files
+during extraction:
+
+The versioned helper creates a fresh current-state backup, stops the timer and
+server, validates archive paths, restores the save, and starts both services:
+
+```sh
+sudo ./docs/project-zomboid-restore.sh \
+  /var/lib/project-zomboid/backups/archive/<archive>.tar.zst
+```
+
+It writes a rollback archive named
+`project-zomboid-<serverName>-pre-restore-<timestamp>.tar.zst` before changing
+the save.
+
+```sh
+systemctl stop project-zomboid.service
+tar --zstd --no-same-owner --no-same-permissions \
+  -xf /var/lib/project-zomboid/backups/archive/<archive>.tar.zst \
+  -C /var/lib/project-zomboid
+chown -R project-zomboid:project-zomboid /var/lib/project-zomboid/Zomboid
+systemctl start project-zomboid.service
+```
+
+Re-provision password files and secret-backed INI values before starting.
+Verify the restored save and server name before allowing players to reconnect.
@@ -0,0 +1,143 @@
+#!/bin/sh
+set -eu
+
+SERVER_NAME=${SERVER_NAME:-servertest}
+DATA_DIR=${DATA_DIR:-/var/lib/project-zomboid}
+ARCHIVE=${1:-}
+
+usage() {
+  printf '%s\n' "Usage: $0 /path/to/project-zomboid-${SERVER_NAME}-<timestamp>.tar.zst"
+  printf '%s\n' "Environment: SERVER_NAME, DATA_DIR"
+}
+
+if [ "$(id -u)" -ne 0 ]; then
+  printf '%s\n' 'Run as root.' >&2
+  exit 1
+fi
+
+if [ -z "$ARCHIVE" ]; then
+  usage >&2
+  exit 2
+fi
+
+if [ ! -r "$ARCHIVE" ]; then
+  printf 'Backup archive is not readable: %s\n' "$ARCHIVE" >&2
+  exit 1
+fi
+
+ARCHIVE_DIR="$DATA_DIR/backups/archive"
+SAVE_DIR="$DATA_DIR/Zomboid/Saves/Multiplayer/$SERVER_NAME"
+SERVER_DIR="$DATA_DIR/Zomboid/Server"
+TMP_LIST=$(mktemp)
+ROLLBACK_ARCHIVE=''
+SERVER_STOPPED=0
+RESTORE_SUCCEEDED=0
+
+cleanup() {
+  rm -f "$TMP_LIST"
+}
+
+on_exit() {
+  status=$?
+  if [ "$status" -ne 0 ] && [ "$SERVER_STOPPED" -eq 1 ] \
+    && [ "$RESTORE_SUCCEEDED" -eq 0 ] && [ -n "$ROLLBACK_ARCHIVE" ]; then
+    set +e
+    rm -rf "$SAVE_DIR"
+    rm -f \
+      "$SERVER_DIR/${SERVER_NAME}_SandboxVars.lua" \
+      "$SERVER_DIR/${SERVER_NAME}_spawnpoints.lua" \
+      "$SERVER_DIR/${SERVER_NAME}_spawnregions.lua"
+    tar --zstd --no-same-owner --no-same-permissions \
+      -xpf "$ROLLBACK_ARCHIVE" -C "$DATA_DIR"
+    chown -R project-zomboid:project-zomboid "$SAVE_DIR" "$SERVER_DIR"
+    systemctl start project-zomboid.service
+    systemctl start project-zomboid-backup.timer
+    printf '%s\n' "Restore failed; current state restored from $ROLLBACK_ARCHIVE" >&2
+  fi
+  cleanup
+  exit "$status"
+}
+trap on_exit EXIT
+
+if ! tar --zstd -tf "$ARCHIVE" >"$TMP_LIST"; then
+  printf 'Archive integrity check failed: %s\n' "$ARCHIVE" >&2
+  exit 1
+fi
+
+while IFS= read -r member; do
+  case "$member" in
+    Zomboid/*) ;;
+    *)
+      printf 'Unsafe archive member: %s\n' "$member" >&2
+      exit 1
+      ;;
+  esac
+  case "$member" in
+    /*|*../*)
+      printf 'Path traversal member: %s\n' "$member" >&2
+      exit 1
+      ;;
+  esac
+done <"$TMP_LIST"
+
+if ! systemctl start project-zomboid-backup.service; then
+  printf '%s\n' 'Could not create fresh backup of current state.' >&2
+  exit 1
+fi
+
+CURRENT_ARCHIVE=$(find "$ARCHIVE_DIR" -maxdepth 1 -type f \
+  -name "project-zomboid-$SERVER_NAME-*.tar.zst" \
+  -printf '%T@ %p\n' | sort -nr | awk 'NR == 1 {sub(/^[^ ]* /, ""); print}')
+
+if [ -z "$CURRENT_ARCHIVE" ]; then
+  printf '%s\n' 'Fresh current-state backup was not found.' >&2
+  exit 1
+fi
+
+stamp=$(date -u +%Y%m%dT%H%M%SZ)
+ROLLBACK_ARCHIVE="$ARCHIVE_DIR/project-zomboid-$SERVER_NAME-pre-restore-$stamp.tar.zst"
+cp --reflink=auto "$CURRENT_ARCHIVE" "$ROLLBACK_ARCHIVE" 2>/dev/null \
+  || cp "$CURRENT_ARCHIVE" "$ROLLBACK_ARCHIVE"
+chmod 0600 "$ROLLBACK_ARCHIVE"
+chown project-zomboid:project-zomboid "$ROLLBACK_ARCHIVE"
+
+systemctl stop project-zomboid-backup.timer
+systemctl stop project-zomboid.service
+SERVER_STOPPED=1
+
+if [ "$(systemctl show project-zomboid --property=ActiveState --value)" != inactive ]; then
+  printf '%s\n' 'Project Zomboid did not stop; refusing to restore.' >&2
+  exit 1
+fi
+
+rm -rf "$SAVE_DIR"
+rm -f \
+  "$SERVER_DIR/${SERVER_NAME}_SandboxVars.lua" \
+  "$SERVER_DIR/${SERVER_NAME}_spawnpoints.lua" \
+  "$SERVER_DIR/${SERVER_NAME}_spawnregions.lua"
+
+tar --zstd --no-same-owner --no-same-permissions \
+  -xpf "$ARCHIVE" -C "$DATA_DIR"
+chown -R project-zomboid:project-zomboid "$SAVE_DIR" "$SERVER_DIR"
+
+systemctl start project-zomboid.service
+started=0
+for _ in $(seq 1 90); do
+  if [ "$(systemctl show project-zomboid --property=ActiveState --value)" = active ] \
+    && [ "$(systemctl show project-zomboid --property=SubState --value)" = running ]; then
+    started=1
+    break
+  fi
+  sleep 2
+done
+
+if [ "$started" -ne 1 ]; then
+  printf 'Restore completed, but service did not become healthy. Rollback: %s\n' \
+    "$ROLLBACK_ARCHIVE" >&2
+  exit 1
+fi
+
+systemctl start project-zomboid-backup.timer
+RESTORE_SUCCEEDED=1
+printf 'Restore completed.\n'
+printf 'Rollback archive: %s\n' "$ROLLBACK_ARCHIVE"
@@ -326,7 +326,7 @@
    "hectic-landing": {
      "inputs": {
        "nixpkgs": [
-          "nixpkgs-fixed"
+          "nixpkgs"
        ]
      },
      "locked": {
@@ -346,7 +346,7 @@
    "home-manager": {
      "inputs": {
        "nixpkgs": [
-          "nixpkgs-fixed"
+          "nixpkgs"
        ]
      },
      "locked": {
@@ -625,6 +625,28 @@
        "type": "github"
      }
    },
+    "iana-angl": {
+      "inputs": {
+        "nixpkgs": [
+          "nixpkgs"
+        ],
+        "rust-overlay": "rust-overlay"
+      },
+      "locked": {
+        "lastModified": 1789498292,
+        "narHash": "sha256-bsrT7MWTXL+tpwDZmI5xWLPZZPYAzU1WjcGlNmqpePw=",
+        "ref": "refs/heads/master",
+        "rev": "2937d257d601b40de2437a51ebc3c7b61b40f679",
+        "revCount": 37,
+        "type": "git",
+        "url": "https://gitea.hectic-lab.com/yukkop/learning.git"
+      },
+      "original": {
+        "rev": "2937d257d601b40de2437a51ebc3c7b61b40f679",
+        "type": "git",
+        "url": "https://gitea.hectic-lab.com/yukkop/learning.git"
+      }
+    },
    "impermanence": {
      "locked": {
        "lastModified": 1737831083,
@@ -671,15 +693,15 @@
    "mechabellum-replay-analysis": {
      "inputs": {
        "nixpkgs": [
-          "nixpkgs-fixed"
+          "nixpkgs"
        ]
      },
      "locked": {
-        "lastModified": 1779576166,
-        "narHash": "sha256-5bSuXkQs7KdbaYwDTdwUFlqOccVjPI2y42TZVq8lsNg=",
+        "lastModified": 1780905541,
+        "narHash": "sha256-hxaKZTcowCDF5RfcCZIWpRY9/ZMm2zJyInNnovBayRg=",
        "ref": "refs/heads/master",
-        "rev": "f00295225c0dade61fe18b32262970c2665fb5fe",
-        "revCount": 110,
+        "rev": "6f1f292db325145bbdf4d0452ce16963c07ecdb1",
+        "revCount": 123,
        "type": "git",
        "url": "ssh://git@github.com/LysmiMx/mechabellum-replay-analysis.git"
      },
@@ -688,6 +710,27 @@
        "url": "ssh://git@github.com/LysmiMx/mechabellum-replay-analysis.git"
      }
    },
+    "nix-darwin": {
+      "inputs": {
+        "nixpkgs": [
+          "nixpkgs"
+        ]
+      },
+      "locked": {
+        "lastModified": 1772129556,
+        "narHash": "sha256-Utk0zd8STPsUJPyjabhzPc5BpPodLTXrwkpXBHYnpeg=",
+        "owner": "nix-darwin",
+        "repo": "nix-darwin",
+        "rev": "ebec37af18215214173c98cf6356d0aca24a2585",
+        "type": "github"
+      },
+      "original": {
+        "owner": "nix-darwin",
+        "ref": "nix-darwin-25.11",
+        "repo": "nix-darwin",
+        "type": "github"
+      }
+    },
    "nix-minecraft": {
      "inputs": {
        "flake-compat": "flake-compat_3",
@@ -872,29 +915,13 @@
        "type": "github"
      }
    },
-    "nixpkgs-fixed": {
-      "locked": {
-        "lastModified": 1771419570,
-        "narHash": "sha256-bxAlQgre3pcQcaRUm/8A0v/X8d2nhfraWSFqVmMcBcU=",
-        "owner": "NixOS",
-        "repo": "nixpkgs",
-        "rev": "6d41bc27aaf7b6a3ba6b169db3bd5d6159cfaa47",
-        "type": "github"
-      },
-      "original": {
-        "owner": "NixOS",
-        "ref": "nixos-25.11",
-        "repo": "nixpkgs",
-        "type": "github"
-      }
-    },
    "nixpkgs_2": {
      "locked": {
-        "lastModified": 1771419570,
-        "narHash": "sha256-bxAlQgre3pcQcaRUm/8A0v/X8d2nhfraWSFqVmMcBcU=",
+        "lastModified": 1779796641,
+        "narHash": "sha256-ZsIrKmhp4vbBXoXXmR/tBXA/UCsAQiJL9vsgZEduhVY=",
        "owner": "NixOS",
        "repo": "nixpkgs",
-        "rev": "6d41bc27aaf7b6a3ba6b169db3bd5d6159cfaa47",
+        "rev": "25f538306313eae3927264466c70d7001dcea1df",
        "type": "github"
      },
      "original": {
@@ -908,7 +935,7 @@
      "inputs": {
        "flake-parts": "flake-parts_2",
        "nixpkgs": [
-          "nixpkgs-fixed"
+          "nixpkgs"
        ],
        "nuschtosSearch": "nuschtosSearch",
        "systems": "systems_5"
@@ -981,32 +1008,54 @@
        "hectic-landing": "hectic-landing",
        "home-manager": "home-manager",
        "hyprland": "hyprland",
+        "iana-angl": "iana-angl",
        "impermanence": "impermanence",
        "mechabellum-replay-analysis": "mechabellum-replay-analysis",
+        "nix-darwin": "nix-darwin",
        "nix-minecraft": "nix-minecraft",
        "nixos-anywhere": "nixos-anywhere",
        "nixos-hardware": "nixos-hardware",
        "nixos-mailserver": "nixos-mailserver",
        "nixos-wsl": "nixos-wsl",
        "nixpkgs": "nixpkgs_2",
-        "nixpkgs-fixed": "nixpkgs-fixed",
        "nixvim": "nixvim",
-        "rust-overlay": "rust-overlay",
+        "rust-overlay": "rust-overlay_2",
        "sops-nix": "sops-nix"
      }
    },
    "rust-overlay": {
      "inputs": {
        "nixpkgs": [
+          "iana-angl",
          "nixpkgs"
        ]
      },
      "locked": {
-        "lastModified": 1738290352,
-        "narHash": "sha256-YKOHUmc0Clm4tMV8grnxYL4IIwtjTayoq/3nqk0QM7k=",
+        "lastModified": 1789457514,
+        "narHash": "sha256-Aggle++fTyAifBy+QBPxjM+obO5iepKW/8MDxQtgGvI=",
        "owner": "oxalica",
        "repo": "rust-overlay",
-        "rev": "b031b584125d33d23a0182f91ddbaf3ab4880236",
+        "rev": "89e99bf0778a8f2cd18c9360c3f19c1ee47fc739",
+        "type": "github"
+      },
+      "original": {
+        "owner": "oxalica",
+        "repo": "rust-overlay",
+        "type": "github"
+      }
+    },
+    "rust-overlay_2": {
+      "inputs": {
+        "nixpkgs": [
+          "nixpkgs"
+        ]
+      },
+      "locked": {
+        "lastModified": 1780629589,
+        "narHash": "sha256-oHysjxZdaEqkmyDpN8G1bl3V+r9uyRD1O66bH0bq0Cs=",
+        "owner": "oxalica",
+        "repo": "rust-overlay",
+        "rev": "7a5a1c0a5cb86a28224304309b68f050835fd1f6",
        "type": "github"
      },
      "original": {
@@ -1,8 +1,18 @@
 {
  description = "yukkop's nix utilities";
+
+  nixConfig = {
+    extra-substituters = [
+      "https://cache.nixos.org"
+      "https://cache.hectic-lab.com/hectic"
+    ];
+    extra-trusted-public-keys = [
+      "hectic:KMQsKow4SoA9K2vOJlOljmx7/Zpf91Yy+5qEtxDDCzA="
+    ];
+  };
+
  inputs = {
    nixpkgs.url = "github:NixOS/nixpkgs/nixos-25.11";
-    nixpkgs-fixed.url = "github:NixOS/nixpkgs/nixos-25.11";
    rust-overlay = {
      url = "github:oxalica/rust-overlay";
      inputs = {
@@ -19,7 +29,7 @@
    };
    nixvim = {
      url = "github:nix-community/nixvim/nixos-25.11";
-      inputs.nixpkgs.follows = "nixpkgs-fixed";
+      inputs.nixpkgs.follows = "nixpkgs";
    };
    disko = {
      url = "github:nix-community/disko";
@@ -30,7 +40,11 @@
    };
    home-manager = {
      url = "github:nix-community/home-manager/release-25.11";
-      inputs.nixpkgs.follows = "nixpkgs-fixed";
+      inputs.nixpkgs.follows = "nixpkgs";
+    };
+    nix-darwin = {
+      url = "github:nix-darwin/nix-darwin/nix-darwin-25.11";
+      inputs.nixpkgs.follows = "nixpkgs";
    };
    nixos-wsl = {
      url = "github:nix-community/NixOS-WSL";
@@ -56,20 +70,23 @@
      # NOTE(yukkop): private repo - SSH access required.
      # Only evaluated when nixosConfigurations."hectic-lab|x86_64-linux" is built.
      url = "git+ssh://git@github.com/liquizz/hectic-landing.git";
-      inputs.nixpkgs.follows = "nixpkgs-fixed";
+      inputs.nixpkgs.follows = "nixpkgs";
+    };
+    iana-angl = {
+      url = "git+https://gitea.hectic-lab.com/yukkop/learning.git?rev=2937d257d601b40de2437a51ebc3c7b61b40f679";
+      inputs.nixpkgs.follows = "nixpkgs";
    };
    mechabellum-replay-analysis = {
      # NOTE(yukkop): private repo - SSH access required.
      # Only evaluated when nixosConfigurations."hectic-lab|x86_64-linux" is built.
      url = "git+ssh://git@github.com/LysmiMx/mechabellum-replay-analysis.git";
-      inputs.nixpkgs.follows = "nixpkgs-fixed";
+      inputs.nixpkgs.follows = "nixpkgs";
    };
  };

  outputs = {
    self,
    nixpkgs,
-    rust-overlay,
    ...
  }@inputs: let
    flake    = ./.;
@@ -82,13 +99,13 @@
        legacyPackages = import ./legacy { inherit system pkgs self; };
        pkgs = import nixpkgs {
          inherit system;
-          overlays = [ (import rust-overlay) ];
+          overlays = [ (import inputs.rust-overlay) ];
        };
      in
        baseOverlay // legacyPackages;
    
    overlays = [ self.overlays.default ];
-  in self-lib.forAllSystemsWithPkgs ([(import rust-overlay)] ++ overlays) ({
+  in self-lib.forAllSystemsWithPkgs ([(import inputs.rust-overlay)] ++ overlays) ({
    system,
    pkgs,
  }: {
@@ -109,8 +126,12 @@
      # FIXME(yukkop): some why I cannot merge nixosConfigurations from `forAllSystemsWithPkgs` with this 
      "neuro|x86_64-linux"     = import ./nixos/system/neuro       { inherit flake self inputs; system = "x86_64-linux"; };
      "games|x86_64-linux"     = import ./nixos/system/games       { inherit flake self inputs; system = "x86_64-linux"; };
-      "wsl|x86_64-linux"       = import ./nixos/system/wsl         { inherit flake self inputs; system = "x86_64-linux"; };
-      "hectic-lab|x86_64-linux" = import ./nixos/system/hectic-lab { inherit flake self inputs; system = "x86_64-linux"; };
+      "wsl|x86_64-linux"        = import ./nixos/system/wsl         { inherit flake self inputs; system = "x86_64-linux"; };
+      "tenix|x86_64-linux"      = import ./nixos/system/tenix       { inherit flake self inputs; system = "x86_64-linux"; };
+      "hectic-lab|x86_64-linux" = import ./nixos/system/hectic-lab  { inherit flake self inputs; system = "x86_64-linux"; };
+    };
+    darwinConfigurations = {
+      "yukkop|aarch64-darwin" = import ./darwin/system/yukkop { inherit flake self inputs; system = "aarch64-darwin"; };
    };
  };
 }
@@ -0,0 +1,30 @@
+{ pkgs, ... }: {
+  programs.tmux = {
+    enable = true;
+    plugins = with pkgs.tmuxPlugins; [ resurrect continuum ];
+    keyMode = "vi";
+    escapeTime = 500;
+    historyLimit = 50000;
+    newSession = true;
+    extraConfig = ''
+      # resurrect
+      set -g @resurrect-strategy-vim 'session'
+      set -g @resurrect-strategy-nvim 'session'
+      set -g @resurrect-capture-pane-contents 'on'
+
+      resurrect_dir="$HOME/.tmux/resurrect"
+      set -g @resurrect-dir $resurrect_dir
+      set -g @resurrect-hook-post-save-all 'target=$(readlink -f $resurrect_dir/last); sed "s| --cmd .*-vim-pack-dir||g; s|/etc/profiles/per-user/$USER/bin/||g; s|/home/$USER/.nix-profile/bin/||g" $target | sponge $target'
+
+      # continuum
+      set -g @continuum-restore 'on'
+      set -g @continuum-boot 'on'
+      set -g @continuum-save-interval '10'
+
+      bind-key    -T copy-mode-vi v                  send-keys -X begin-selection
+      bind-key    -T copy-mode-vi C-v                send-keys -X rectangle-toggle
+
+      bind-key O select-pane -t :.-
+    '';
+  };
+}
@@ -0,0 +1,2 @@
+__pycache__/
+*.py[cod]
@@ -0,0 +1,162 @@
+# Attic repack migration helper
+
+Local-only operator tool for safe resumable Attic cache repack/migration. Parent automation starts old/new services, supplies secrets, seeds spool, and runs this CLI.
+
+## Deployment layout
+
+### Operational state — 2026-09-10
+
+The primary `/hectic` endpoint now serves `nix-cache-hectic-lab` through
+`atticd-repacked`; the cutover was applied with NixOS `switch`. The original
+bucket/database remain preserved and readable at `/previous/hectic`.
+`/next/hectic` is an alias for the new backend. Existing public keys and CI
+tokens remain valid, and the primary endpoint is writable again.
+
+The migrated inventory contains 1343 paths and 1195 unique NAR hashes. The
+independent inventory comparison and aggregate full-read receipts are recorded
+in `/var/lib/attic-repack/verification-receipt.json`. Transient S3 504/read errors
+required retries; this is data-integrity evidence, not a claim that Hetzner's
+read availability is fixed.
+
+All migration, verification, seeding, and watcher jobs have been stopped for
+user-controlled load testing. Do not automatically restart bulk verification.
+The transport-fix generation is
+`/nix/store/mjx3gzrh03vq22j5nmrzwa56kfhdn702-nixos-system-hectic-lab-25.11.20260526.25f5383`
+with the HTTP/1.1 S3 transport and upload-proxy timeout correction. Check
+`/run/current-system` to confirm the active deployment.
+The pinned rollback generation remains at
+`/var/lib/attic-repack/rollback-system`; backups remain private under
+`/var/lib/attic-repack/backups` and include the cache signing key.
+
+- Original backend: `atticd`, port 8081, `/var/lib/atticd/server.db`, bucket
+  `cache-hectic-lab` in HEL1.
+- During the write freeze and after cutover the original backend runs in
+  `api-server` mode, without its garbage collector, to preserve the comparison
+  dataset. Public write methods remain blocked by nginx.
+- Repacked backend: `atticd-repacked`, port 8082,
+  `/var/lib/atticd-repacked/server.db`, bucket `nix-cache-hectic-lab` in HEL1.
+- Both use the same `hectic` signing key and existing JWT verification secret;
+  clients do not need a new trusted public key or token.
+- New chunk settings: threshold/minimum 1 MiB, average 2 MiB, maximum 4 MiB.
+- `https://cache.hectic-lab.com/next/hectic` selects the new backend.
+- `https://cache.hectic-lab.com/previous/hectic` selects the original backend;
+  nginx permits GET/HEAD only there.
+- `repackedActive` in `nixos/system/hectic-lab/attic.nix` selects which backend
+  owns the original `/hectic` URL. Keep it false until all cutover gates pass.
+
+## Cutover and rollback gates
+
+1. Finish all migration partitions, then run an unfiltered migration/delta pass.
+2. Confirm no CI writers remain. Set `migrationWriteFreeze = true` while
+   `repackedActive = false`, apply the small NixOS change, and briefly stop the
+   original Attic to drain/cancel any prior in-flight writes.
+3. Take a SQLite backup with SQLite's backup API, not a raw live-file copy.
+   Keep backups and manifests under private `/var/lib/attic-repack`; the SQLite
+   backup includes the cache's private signing key.
+4. Restart the original backend for reads only, refresh the complete inventory,
+   migrate any final delta, then run unfiltered `verify` across all 1343 old
+   paths. Its exit status must be zero with zero exhausted payload verification
+   failures; independently compare old/new store-path, NAR hash, size and
+   metadata inventories from the databases. `status` alone is not a cutover
+   certificate.
+5. Pin the old/staging NixOS generation as a GC root, set `repackedActive = true`,
+   build, inspect dry activation, and switch. `/hectic` now reaches the new
+   backend; old data and `/previous/hectic` remain available.
+6. Test public reads, signatures, and an authenticated upload at the original
+   URL. Do not remove the old bucket or database as part of this procedure.
+
+Rollback reapplies the pinned staging generation. The new backend and its data
+must remain preserved: paths first uploaded after cutover may exist only there.
+When editing the flags manually, clear `migrationWriteFreeze` explicitly if
+writes to the original backend are intended after rollback.
+
+## Throughput comparison
+
+Compare the same store-path hashes at `/next/hectic` and `/previous/hectic` with
+the same request concurrency. For example, fetch
+`https://cache.hectic-lab.com/next/hectic/nar/<store-path-hash>.nar` with
+`curl --fail --location --output /dev/null --write-out 'bytes=%{size_download} seconds=%{time_total}\n'`.
+Do not print effective redirect URLs: S3 redirects contain temporary signatures.
+Compare wall time and error rate as well as bytes/second because compressed sizes
+can differ after rechunking. Do not use a build with source fallback as a pure
+cache throughput measurement. The two endpoints share the VPS and nginx, so run
+the comparison sequentially or account for shared-resource contention.
+
+## Spool/state convention
+
+Default state dir: `/var/lib/attic-repack` (`0700`). Raw NAR spool path:
+
+```text
+/var/lib/attic-repack/raw/{sha256hex}.nar
+```
+
+Parent may seed this file directly. Tool always verifies SHA-256 and byte length before upload. Checkpoints live under `checkpoints/{store_path_hash}.json` and contain no keypair/token. Forced payload verification records a receipt with `payload_verified_at`, `payload_verify_attempts`, `payload_sha256`, and `payload_bytes` only after a complete successful read.
+
+## Commands
+
+```sh
+attic-repack init \
+  --old-db file:/var/lib/atticd/server.db?mode=ro \
+  --old-url http://127.0.0.1:8081 \
+  --new-url http://127.0.0.1:8082 \
+  --host cache.hectic-lab.com \
+  --atticadm /run/current-system/sw/bin/atticadm \
+  --server-config /etc/atticd/server.toml
+
+attic-repack inventory --state-dir /var/lib/attic-repack > inventory.json
+attic-repack status --state-dir /var/lib/attic-repack
+attic-repack migrate --state-dir /var/lib/attic-repack --workers 2 --limit 20
+attic-repack verify --state-dir /var/lib/attic-repack --workers 2
+```
+
+`ATTIC_MIGRATION_TOKEN` may be set for manual/tests. Otherwise token is minted in memory with `atticadm make-token` for hectic pull/push/create-cache/configure-cache. Token/keypair are never printed.
+
+## Inventory JSON
+
+`inventory` writes `attic-repack-inventory-v1`:
+
+```json
+{
+  "format": "attic-repack-inventory-v1",
+  "cache": "hectic",
+  "spool_dir": "/var/lib/attic-repack/raw",
+  "raw_nar_filename": "{sha256hex}.nar",
+  "records": [
+    {"nar_hash":"sha256:...","nar_size":123,"store_path":"/nix/store/...","metadata_fingerprint":"..."}
+  ]
+}
+```
+
+Records also include upload metadata: `store_path_hash`, `references`, `system`, `deriver`, `sigs`, `ca`.
+
+## Safety
+
+- Checkpoints and `status` are progress information, not a final cutover proof.
+  After stopping old writers and taking a consistent snapshot, run an unfiltered
+  `verify` (no `--paths-file` or `--limit`) to reread every new NAR and reconcile
+  all paths, metadata, hashes, and sizes before switching the primary endpoint.
+  Do not skip files, change expected hashes, or relax server/client timeouts to
+  pass this gate.
+- A local store path can differ from the historical cached NAR. Such a local
+  copy is rejected and recovered from the original S3 chunks instead.
+- Old DB is opened readonly; old SQL NAR/chunk tables are never copied.
+- Missing local raw NARs are reconstructed from old S3 chunkrefs with per-object retries and chunk/full hash checks.
+- Upload uses Attic `PUT /_api/v1/upload-path` with JSON preamble plus raw uncompressed NAR.
+- New cache verification compares immutable metadata against old rendered narinfo
+  and reads/decompresses one payload per verified path invocation. Payload reads
+  make up to three fresh attempts for transport HTTP 408/429/5xx and truncated
+  body/decompressor EOF failures only. Each attempt follows a new GET/redirect,
+  starts SHA-256 and byte counts from zero, closes failed readers, and fails
+  immediately on hash mismatch, full-size mismatch, oversized payload, missing
+  URL, unsupported compression, or HTTP 4xx other than 408/429.
+- A receipt with retries proves the path was fully read and matched integrity; it
+  does not prove the storage provider is healthy. Treat retry events as provider
+  health signals separate from cutover correctness.
+- Authenticated HTTP is refused unless URL host is loopback.
+
+## Local build/test
+
+```sh
+nix build --option eval-cache false --impure --expr "let flake = builtins.getFlake \"git+file://$PWD\"; pkgs = import flake.inputs.nixpkgs { system = builtins.currentSystem; }; in pkgs.callPackage ./infra/attic-migration {}"
+nix build --option eval-cache false --impure --expr "let flake = builtins.getFlake \"git+file://$PWD\"; pkgs = import flake.inputs.nixpkgs { system = builtins.currentSystem; }; p = pkgs.callPackage ./infra/attic-migration {}; in p.passthru.tests.unittest"
+```
@@ -0,0 +1,48 @@
+{ pkgs }:
+
+let
+  source = pkgs.lib.cleanSourceWith {
+    src = ./.;
+    filter = path: type:
+      builtins.baseNameOf path != "__pycache__"
+      && !(pkgs.lib.hasSuffix ".pyc" path);
+  };
+  pythonEnv = pkgs.python3.withPackages (ps: [
+    ps.requests
+    ps.boto3
+    ps.zstandard
+  ]);
+in
+pkgs.stdenv.mkDerivation {
+  pname = "attic-repack";
+  version = "0.1.0";
+  src = source;
+
+  nativeBuildInputs = [ pkgs.makeWrapper ];
+
+  installPhase = ''
+    mkdir -p $out/bin $out/libexec/attic-repack
+    cp $src/repack.py $out/libexec/attic-repack/repack.py
+    chmod +x $out/libexec/attic-repack/repack.py
+    makeWrapper ${pythonEnv}/bin/python3 $out/bin/attic-repack \
+      --add-flags $out/libexec/attic-repack/repack.py \
+      --prefix PATH : ${pkgs.lib.makeBinPath [ pkgs.nix ]}
+  '';
+
+  doCheck = true;
+  checkPhase = ''
+    ${pythonEnv}/bin/python3 -m unittest discover -s $src -p 'test_*.py'
+  '';
+
+  passthru = {
+    inherit pythonEnv;
+    tests.unittest = pkgs.runCommand "attic-repack-unittest" {
+      nativeBuildInputs = [ pythonEnv pkgs.nix ];
+    } ''
+      cp -r ${source} ./src
+      chmod -R u+w ./src
+      ${pythonEnv}/bin/python3 -m unittest discover -s ./src -p 'test_*.py'
+      mkdir -p $out
+    '';
+  };
+}
@@ -0,0 +1,562 @@
+import argparse
+import hashlib
+import http.server
+import io
+import json
+import os
+import pathlib
+import sqlite3
+import tempfile
+import threading
+import time
+import unittest
+from unittest import mock
+
+import repack
+
+
+def sha(data):
+    return hashlib.sha256(data).hexdigest()
+
+
+class FakeResponse:
+    def __init__(self, status_code=200, content=b"", json_data=None, raw=None, headers=None):
+        self.status_code = status_code
+        self.content = content
+        self._json = json_data
+        self.raw = raw or io.BytesIO(content)
+        self.headers = headers or {}
+        self.close_count = 0
+
+    def json(self):
+        return self._json
+
+    def close(self):
+        self.close_count += 1
+
+
+class FakeSession:
+    def __init__(self):
+        self.calls = []
+        self.routes = {}
+
+    def request(self, method, url, **kwargs):
+        self.calls.append((method, url, kwargs))
+        key = (method, pathlib.PurePosixPath(url.split("?", 1)[0]).as_posix())
+        response = self.routes.get(key) or self.routes.get((method, url))
+        if callable(response):
+            return response(method, url, kwargs)
+        return response or FakeResponse(404)
+
+
+class ThreadedHTTP:
+    def __init__(self, handler):
+        self.server = http.server.ThreadingHTTPServer(("127.0.0.1", 0), handler)
+        self.thread = threading.Thread(target=self.server.serve_forever, daemon=True)
+
+    @property
+    def url(self):
+        host, port = self.server.server_address[:2]
+        return f"http://{host}:{port}"
+
+    def __enter__(self):
+        self.thread.start()
+        return self
+
+    def __exit__(self, *_args):
+        self.server.shutdown()
+        self.thread.join(timeout=5)
+        self.server.server_close()
+
+
+class RepackTests(unittest.TestCase):
+    def test_options_before_subcommand_are_preserved(self):
+        parser = repack.build_parser()
+        args = parser.parse_args(["--atticadm", "/safe/atticadm", "--server-config", "/safe/config", "--workers", "1", "init"])
+        self.assertEqual(args.atticadm, "/safe/atticadm")
+        self.assertEqual(args.server_config, "/safe/config")
+        self.assertEqual(args.workers, 1)
+        args = parser.parse_args(["--workers", "1", "migrate", "--workers", "2"])
+        self.assertEqual(args.workers, 2)
+
+    def test_inventory_reads_sqlite_reserved_references_column(self):
+        with tempfile.TemporaryDirectory() as td:
+            db = pathlib.Path(td) / "old.db"
+            con = sqlite3.connect(db)
+            con.executescript('''
+                CREATE TABLE cache(id INTEGER, name TEXT, deleted_at TEXT);
+                CREATE TABLE nar(id INTEGER, nar_hash TEXT, nar_size INTEGER, state TEXT);
+                CREATE TABLE object(cache_id INTEGER, nar_id INTEGER,
+                    store_path_hash TEXT, store_path TEXT, "references" TEXT,
+                    system TEXT, deriver TEXT, sigs TEXT, ca TEXT);
+                INSERT INTO cache VALUES(1, 'hectic', NULL);
+            ''')
+            con.execute("INSERT INTO nar VALUES(1, ?, 7, 'V')", ("sha256:" + "a" * 64,))
+            con.execute("INSERT INTO object VALUES(1, 1, ?, ?, ?, NULL, NULL, ?, NULL)",
+                        ("b" * 32, "/nix/store/" + "b" * 32 + "-test", '["dependency"]', '[]'))
+            con.commit()
+            con.close()
+            rows = repack.InventoryDB(str(db), "hectic").records()
+            self.assertEqual(len(rows), 1)
+            self.assertEqual(rows[0]["references"], ["dependency"])
+
+    def test_metadata_import_keypair_capital(self):
+        with tempfile.TemporaryDirectory() as td:
+            db = pathlib.Path(td) / "old.db"
+            con = sqlite3.connect(db)
+            con.executescript("""
+            create table cache(id integer primary key,name text,keypair text,is_public integer,store_dir text,priority integer,upstream_cache_key_names text,retention_period integer,deleted_at text);
+            insert into cache values(1,'hectic','priv',1,'/nix/store',30,'["up"]',3600,null);
+            """)
+            con.close()
+            args = self.args(td, old_db=str(db))
+            mig = repack.Migrator(args)
+            old = mock.Mock()
+            old.get_cache_config.return_value = {"public_key": "pub"}
+            new = mock.Mock()
+            new.get_cache_config.side_effect = [None, {"public_key": "pub"}]
+            mig.old_client = old
+            mig.new_client = new
+            mig.init_cache()
+            body = new.create_cache.call_args.args[0]
+            self.assertEqual(body["keypair"], {"Keypair": "priv"})
+            new.patch_retention.assert_called_with({"Period": 3600})
+
+    def test_bad_hash_rejects_raw_spool(self):
+        with tempfile.TemporaryDirectory() as td:
+            args = self.args(td)
+            mig = repack.Migrator(args)
+            record = self.record(b"good")
+            raw = mig.state.raw_path(repack.nar_hash_hex(record["nar_hash"]))
+            raw.write_bytes(b"bad")
+            with self.assertRaises(repack.RepackError):
+                mig.ensure_raw_nar(record)
+
+    def test_local_mismatch_recovers_original_from_old_s3(self):
+        with tempfile.TemporaryDirectory() as td:
+            original = b"original cached NAR"
+            record = self.record(original)
+            local = pathlib.Path(td) / "different-local-copy"
+            local.write_bytes(b"different")
+            record["store_path"] = str(local)
+            mig = repack.Migrator(self.args(td))
+
+            def recover(_record, path):
+                path.write_bytes(original)
+
+            with mock.patch.object(mig, "dump_local_store_path", side_effect=repack.RepackError("nix dump-path NAR hash/size mismatch")), \
+                    mock.patch.object(mig.s3, "assemble", side_effect=recover) as assemble:
+                result = mig.ensure_raw_nar(record)
+            self.assertEqual(result.read_bytes(), original)
+            assemble.assert_called_once()
+
+    def test_no_compile_subprocess_commands(self):
+        with tempfile.TemporaryDirectory() as td:
+            args = self.args(td)
+            mig = repack.Migrator(args)
+            data = b"nar"
+            record = self.record(data)
+            store = pathlib.Path(record["store_path"])
+            with mock.patch("subprocess.run") as run:
+                def fake_run(cmd, check, stdout, stderr, env):
+                    self.assertEqual(cmd[:3], ["nix", "nar", "pack"])
+                    self.assertNotIn("build", cmd)
+                    self.assertNotIn(repack.TOKEN_ENV, env)
+                    stdout.write(data)
+                    return mock.Mock()
+                run.side_effect = fake_run
+                path = mig.state.raw_path(repack.nar_hash_hex(record["nar_hash"]))
+                mig.dump_local_store_path(record, path)
+                self.assertEqual(path.read_bytes(), data)
+                self.assertTrue(str(store).startswith("/nix/store/"))
+
+    def test_resumable_checkpoint(self):
+        with tempfile.TemporaryDirectory() as td:
+            state = repack.State(pathlib.Path(td))
+            record = self.record(b"abc")
+            state.set_checkpoint(record, "verified", 2)
+            cp = state.get_checkpoint(record)
+            self.assertEqual(cp["status"], "verified")
+            self.assertEqual(cp["tries"], 2)
+            self.assertEqual(cp["metadata_fingerprint"], repack.metadata_fingerprint(record))
+
+    def test_root_backend_upload_preamble(self):
+        data = b"abc"
+        record = self.record(data)
+        with tempfile.TemporaryDirectory() as td:
+            nar = pathlib.Path(td) / "x.nar"
+            nar.write_bytes(data)
+            client = repack.AtticClient("http://127.0.0.1:8082", "hectic", "cache.hectic-lab.com", repack.TokenProvider(None, None, "hectic"))
+            assert client.token_provider is not None
+            client.token_provider._token = "tok"
+            client.token_provider._expires = repack.now() + 3600
+            sess = FakeSession()
+            client._local.session = sess
+            def put(method, url, kwargs):
+                self.assertTrue(url.endswith("/_api/v1/upload-path"))
+                self.assertIn("X-Attic-Nar-Info-Preamble-Size", kwargs["headers"])
+                self.assertEqual(len(kwargs["data"]), int(kwargs["headers"]["Content-Length"]))
+                body = kwargs["data"].read()
+                pre = int(kwargs["headers"]["X-Attic-Nar-Info-Preamble-Size"])
+                meta = json.loads(body[:pre])
+                self.assertEqual(meta["store_path"], record["store_path"])
+                self.assertEqual(body[pre:], data)
+                return FakeResponse(200)
+            sess.routes[("PUT", "http://127.0.0.1:8082/_api/v1/upload-path")] = put
+            client.upload(record, nar)
+
+    def test_real_http_upload_has_content_length_no_chunked(self):
+        try:
+            repack.requests_module()
+        except ModuleNotFoundError:
+            self.skipTest("requests not installed outside Nix test env")
+        data = b"nar-bytes"
+        record = self.record(data)
+        seen = {}
+
+        class Handler(http.server.BaseHTTPRequestHandler):
+            def do_PUT(self):
+                seen["path"] = self.path
+                seen["host"] = self.headers.get("Host")
+                seen["te"] = self.headers.get("Transfer-Encoding")
+                length = int(self.headers["Content-Length"])
+                body = self.rfile.read(length)
+                pre = int(self.headers["X-Attic-Nar-Info-Preamble-Size"])
+                seen["meta"] = json.loads(body[:pre])
+                seen["nar"] = body[pre:]
+                self.send_response(200); self.end_headers()
+
+            def log_message(self, format, *args):
+                pass
+
+        with tempfile.TemporaryDirectory() as td, ThreadedHTTP(Handler) as srv:
+            nar = pathlib.Path(td) / "x.nar"
+            nar.write_bytes(data)
+            tp = repack.TokenProvider(None, None, "hectic")
+            tp._token = "tok"; tp._expires = repack.now() + 3600
+            repack.AtticClient(srv.url, "hectic", "cache.hectic-lab.com", tp).upload(record, nar)
+        self.assertEqual(seen["path"], "/_api/v1/upload-path")
+        self.assertEqual(seen["host"], "cache.hectic-lab.com")
+        self.assertIsNone(seen["te"])
+        self.assertEqual(seen["meta"]["store_path"], record["store_path"])
+        self.assertEqual(seen["nar"], data)
+
+    def test_payload_relative_url_and_redirect_strips_host(self):
+        try:
+            repack.requests_module()
+        except ModuleNotFoundError:
+            self.skipTest("requests not installed outside Nix test env")
+        data = b"nar"
+        seen = {}
+
+        class S3Handler(http.server.BaseHTTPRequestHandler):
+            def do_GET(self):
+                seen["s3_path"] = self.path
+                seen["s3_host"] = self.headers.get("Host")
+                seen["s3_auth"] = self.headers.get("Authorization")
+                self.send_response(200); self.end_headers(); self.wfile.write(data)
+            def log_message(self, format, *args):
+                pass
+
+        with ThreadedHTTP(S3Handler) as s3:
+            class CacheHandler(http.server.BaseHTTPRequestHandler):
+                def do_GET(self):
+                    seen["cache_path"] = self.path
+                    seen["cache_host"] = self.headers.get("Host")
+                    seen["cache_auth"] = self.headers.get("Authorization")
+                    self.send_response(302)
+                    self.send_header("Location", s3.url + "/object")
+                    self.end_headers()
+                def log_message(self, format, *args):
+                    pass
+
+            with ThreadedHTTP(CacheHandler) as cache:
+                client = repack.AtticClient(cache.url, "hectic", "cache.hectic-lab.com", None)
+                client.verify_payload({"URL":"nar/x","Compression":"none"}, sha(data), len(data), cache.url + "/hectic/abcd.narinfo")
+        self.assertEqual(seen["cache_path"], "/hectic/nar/x")
+        self.assertEqual(seen["cache_host"], "cache.hectic-lab.com")
+        self.assertIsNone(seen["cache_auth"])
+        self.assertEqual(seen["s3_path"], "/object")
+        self.assertNotEqual(seen["s3_host"], "cache.hectic-lab.com")
+        self.assertIsNone(seen["s3_auth"])
+
+    def test_payload_retry_truncated_then_full_resets_hash(self):
+        data = b"complete NAR bytes"
+        client = repack.AtticClient("http://127.0.0.1:8082", "hectic", None, None)
+        sess = FakeSession(); client._local.session = sess
+        responses = [FakeResponse(200, content=data[:4]), FakeResponse(200, content=data)]
+
+        def get(_method, _url, _kwargs):
+            return responses.pop(0)
+
+        sess.routes[("GET", "http://127.0.0.1:8082/hectic/nar/x")] = get
+        with mock.patch("time.sleep") as sleep:
+            receipt = client.verify_payload({"URL": "nar/x", "Compression": "none"}, sha(data), len(data), "http://127.0.0.1:8082/hectic/abcd.narinfo", "/nix/store/abcd-name")
+        self.assertEqual(receipt, {"attempts": 2, "sha256": sha(data), "bytes": len(data)})
+        self.assertEqual(len(sess.calls), 2)
+        self.assertEqual(sleep.call_count, 1)
+
+    def test_payload_persistent_timeouts_fail_after_three(self):
+        client = repack.AtticClient("http://127.0.0.1:8082", "hectic", None, None)
+        sess = FakeSession(); client._local.session = sess
+
+        def timeout(_method, _url, _kwargs):
+            raise TimeoutError()
+
+        sess.routes[("GET", "http://127.0.0.1:8082/hectic/nar/x")] = timeout
+        with mock.patch("time.sleep") as sleep, self.assertRaises(TimeoutError):
+            client.verify_payload({"URL": "nar/x", "Compression": "none"}, sha(b"x"), 1, "http://127.0.0.1:8082/hectic/abcd.narinfo")
+        self.assertEqual(len(sess.calls), 3)
+        self.assertEqual(sleep.call_count, 2)
+
+    def test_payload_full_size_wrong_hash_fails_after_one(self):
+        client = repack.AtticClient("http://127.0.0.1:8082", "hectic", None, None)
+        sess = FakeSession(); client._local.session = sess
+        sess.routes[("GET", "http://127.0.0.1:8082/hectic/nar/x")] = FakeResponse(200, content=b"bad")
+        with self.assertRaises(repack.PayloadIntegrityError):
+            client.verify_payload({"URL": "nar/x", "Compression": "none"}, sha(b"nar"), 3, "http://127.0.0.1:8082/hectic/abcd.narinfo")
+        self.assertEqual(len(sess.calls), 1)
+
+    def test_payload_oversize_fails_after_one(self):
+        client = repack.AtticClient("http://127.0.0.1:8082", "hectic", None, None)
+        sess = FakeSession(); client._local.session = sess
+        sess.routes[("GET", "http://127.0.0.1:8082/hectic/nar/x")] = FakeResponse(200, content=b"toolong")
+        with self.assertRaises(repack.PayloadIntegrityError):
+            client.verify_payload({"URL": "nar/x", "Compression": "none"}, sha(b"too"), 3, "http://127.0.0.1:8082/hectic/abcd.narinfo")
+        self.assertEqual(len(sess.calls), 1)
+
+    def test_payload_http_403_no_retry_and_closes(self):
+        client = repack.AtticClient("http://127.0.0.1:8082", "hectic", None, None)
+        sess = FakeSession(); client._local.session = sess
+        resp = FakeResponse(403)
+        sess.routes[("GET", "http://127.0.0.1:8082/hectic/nar/x")] = resp
+        with self.assertRaises(repack.RepackError):
+            client.verify_payload({"URL": "nar/x", "Compression": "none"}, sha(b"x"), 1, "http://127.0.0.1:8082/hectic/abcd.narinfo")
+        self.assertEqual(len(sess.calls), 1)
+        self.assertEqual(resp.close_count, 1)
+
+    def test_auth_api_redirect_refused(self):
+        tp = repack.TokenProvider(None, None, "hectic")
+        tp._token = "tok"; tp._expires = repack.now() + 3600
+        client = repack.AtticClient("http://127.0.0.1:8082", "hectic", None, tp)
+        sess = FakeSession(); client._local.session = sess
+        sess.routes[("GET", "http://127.0.0.1:8082/_api/v1/cache-config/hectic")] = FakeResponse(302, headers={"Location":"http://evil/"})
+        with self.assertRaises(repack.RepackError):
+            client.get_cache_config()
+
+    def test_concatenated_zstd_correct(self):
+        try:
+            zstd = repack.zstd_module()
+        except ModuleNotFoundError:
+            self.skipTest("zstandard not installed outside Nix test env")
+        plain = b"a" * 100 + b"b" * 100
+        cctx = zstd.ZstdCompressor()
+        payload = cctx.compress(plain[:100]) + cctx.compress(plain[100:])
+        narinfo = {"URL": "nar/x.nar.zst", "Compression": "zstd"}
+        client = repack.AtticClient("http://127.0.0.1:8082", "hectic", None, None)
+        sess = FakeSession()
+        client._local.session = sess
+        sess.routes[("GET", "http://127.0.0.1:8082/hectic/nar/x.nar.zst")] = FakeResponse(200, raw=io.BytesIO(payload))
+        client.verify_payload(narinfo, sha(plain), len(plain), "http://127.0.0.1:8082/hectic/abcd.narinfo")
+
+    def test_perchunk_retry_cache(self):
+        try:
+            zstd = repack.zstd_module()
+        except ModuleNotFoundError:
+            self.skipTest("zstandard not installed outside Nix test env")
+        with tempfile.TemporaryDirectory() as td:
+            db = pathlib.Path(td) / "old.db"
+            plain = b"chunk"
+            comp = zstd.ZstdCompressor().compress(plain)
+            con = sqlite3.connect(db)
+            con.executescript("""
+            create table chunkref(nar_id integer,seq integer,chunk_id integer);
+            create table chunk(id integer primary key,state text,chunk_hash text,chunk_size integer,file_hash text,file_size integer,compression text,remote_file text);
+            """)
+            con.execute("insert into chunkref values(1,0,1)")
+            con.execute("insert into chunk values(1,'V',?,?,?,?,?,?)", (sha(plain), len(plain), sha(comp), len(comp), "zstd", json.dumps({"S3":{"region":"hel1","bucket":"cache-hectic-lab","key":"k"}})))
+            con.commit(); con.close()
+            state = repack.State(pathlib.Path(td) / "state")
+            asm = repack.OldS3Assembler(repack.InventoryDB(str(db), "hectic"), state, "https://example", "cache-hectic-lab", "hel1")
+            fake_client = mock.Mock()
+            fake_client.get_object.side_effect = [Exception("once"), {"Body": io.BytesIO(comp)}]
+            out = pathlib.Path(td) / "out.nar"
+            with mock.patch.object(asm, "client", return_value=fake_client):
+                asm.assemble({"nar_id": 1, "nar_hash": "sha256:" + sha(plain), "nar_size": len(plain)}, out)
+            self.assertEqual(out.read_bytes(), plain)
+            self.assertEqual(fake_client.get_object.call_count, 2)
+            fake_client.get_object.reset_mock()
+            self.assertEqual(asm._compressed_chunk(asm.db.chunk_rows(1)[0]), comp)
+            fake_client.get_object.assert_not_called()
+
+    def test_chunk_prefetch_is_bounded_and_preserves_order(self):
+        with tempfile.TemporaryDirectory() as td:
+            pieces = [f"chunk-{i}\n".encode() for i in range(12)]
+            rows = [{"seq": i, "compression": "none", "chunk_hash": sha(data), "chunk_size": len(data)}
+                    for i, data in enumerate(pieces)]
+            db = mock.Mock()
+            db.chunk_rows.return_value = rows
+            asm = repack.OldS3Assembler(db, repack.State(pathlib.Path(td) / "state"), "https://example", "cache-hectic-lab", "hel1")
+            lock = threading.Lock()
+            active = 0
+            peak = 0
+
+            def fetch(row):
+                nonlocal active, peak
+                with lock:
+                    active += 1
+                    peak = max(peak, active)
+                time.sleep(0.04 if row["seq"] == 0 else 0.01)
+                with lock:
+                    active -= 1
+                return pieces[row["seq"]]
+
+            whole = b"".join(pieces)
+            out = pathlib.Path(td) / "result.nar"
+            with mock.patch.object(asm, "_compressed_chunk", side_effect=fetch):
+                asm.assemble({"nar_id": 1, "nar_hash": "sha256:" + sha(whole), "nar_size": len(whole)}, out)
+            self.assertEqual(out.read_bytes(), whole)
+            self.assertGreater(peak, 1)
+            self.assertLessEqual(peak, 4)
+
+    def test_zstd_chunk_no_content_size_concat(self):
+        try:
+            zstd = repack.zstd_module()
+        except ModuleNotFoundError:
+            self.skipTest("zstandard not installed outside Nix test env")
+        cctx = zstd.ZstdCompressor(write_content_size=False)
+        payload = cctx.compress(b"aa") + cctx.compress(b"bb")
+        self.assertEqual(repack.decompress_chunk(payload, "zstd"), b"aabb")
+
+    def test_mismatch_new_key_fails(self):
+        with tempfile.TemporaryDirectory() as td:
+            db = pathlib.Path(td) / "old.db"
+            con = sqlite3.connect(db)
+            con.executescript("""
+            create table cache(id integer primary key,name text,keypair text,is_public integer,store_dir text,priority integer,upstream_cache_key_names text,retention_period text,deleted_at text);
+            insert into cache values(1,'hectic','priv',1,'/nix/store',30,'[]',null,null);
+            """)
+            con.close()
+            mig = repack.Migrator(self.args(td, old_db=str(db)))
+            mig.old_client = mock.Mock(); mig.old_client.get_cache_config.return_value = {"public_key":"old"}
+            mig.new_client = mock.Mock(); mig.new_client.get_cache_config.return_value = {"public_key":"new","is_public":True,"store_dir":"/nix/store","priority":30,"upstream_cache_key_names":[]}
+            with self.assertRaises(repack.RepackError):
+                mig.init_cache()
+
+    def test_init_requires_old_public_key(self):
+        with tempfile.TemporaryDirectory() as td:
+            db = pathlib.Path(td) / "old.db"
+            con = sqlite3.connect(db)
+            con.executescript("""
+            create table cache(id integer primary key,name text,keypair text,is_public integer,store_dir text,priority integer,upstream_cache_key_names text,retention_period text,deleted_at text);
+            insert into cache values(1,'hectic','priv',1,'/nix/store',30,'[]',null,null);
+            """)
+            con.close()
+            mig = repack.Migrator(self.args(td, old_db=str(db)))
+            mig.old_client = mock.Mock(); mig.old_client.get_cache_config.return_value = {}
+            mig.new_client = mock.Mock(); mig.new_client.get_cache_config.return_value = None
+            with self.assertRaises(repack.RepackError):
+                mig.init_cache()
+
+    def test_verify_readonly_no_put(self):
+        with tempfile.TemporaryDirectory() as td:
+            mig = repack.Migrator(self.args(td))
+            record = self.record(b"abc")
+            mig.selected_records = lambda: [record]
+            mig.new_client = mock.Mock()
+            mig.new_client.get_narinfo.return_value = {**repack.expected_narinfo(record), "URL": "nar/x", "Compression": "none"}
+            mig.new_client.verify_payload.return_value = {"attempts": 1, "sha256": sha(b"abc"), "bytes": 3}
+            mig.new_client.narinfo_url.return_value = "http://127.0.0.1:8082/hectic/abcd.narinfo"
+            mig.old_client = mock.Mock()
+            mig.old_client.get_narinfo.return_value = {**repack.expected_narinfo(record)}
+            mig.migrate(True)
+            mig.new_client.upload.assert_not_called()
+            mig.new_client.verify_payload.assert_called_once()
+
+    def test_payload_receipt_only_for_forced_successful_full_read(self):
+        with tempfile.TemporaryDirectory() as td:
+            mig = repack.Migrator(self.args(td))
+            record = self.record(b"abc")
+            mig.selected_records = lambda: [record]
+            narinfo = {**repack.expected_narinfo(record), "URL": "nar/x", "Compression": "none"}
+            old_info = {**repack.expected_narinfo(record)}
+            mig.old_client = mock.Mock(); mig.old_client.get_narinfo.return_value = old_info
+            mig.new_client = mock.Mock(); mig.new_client.get_narinfo.return_value = narinfo; mig.new_client.narinfo_url.return_value = "http://127.0.0.1/hectic/abcd.narinfo"
+            mig.new_client.verify_payload.return_value = {"attempts": 2, "sha256": sha(b"abc"), "bytes": 3}
+            self.assertEqual(mig.migrate(True), 0)
+            cp = mig.state.get_checkpoint(record)
+            self.assertEqual(cp["payload_verify_attempts"], 2)
+            self.assertEqual(cp["payload_sha256"], sha(b"abc"))
+            self.assertEqual(cp["payload_bytes"], 3)
+            self.assertTrue(cp["payload_verified_at"].endswith("Z"))
+            first_verified_at = cp["payload_verified_at"]
+
+            mig.new_client.verify_payload.reset_mock()
+            self.assertEqual(mig.migrate(False), 0)
+            cp = mig.state.get_checkpoint(record)
+            self.assertEqual(cp["payload_verified_at"], first_verified_at)
+            mig.new_client.verify_payload.assert_not_called()
+
+            mig.new_client.get_narinfo.return_value = narinfo
+            mig.new_client.verify_payload.side_effect = repack.PayloadIntegrityError("new NAR payload hash/size mismatch")
+            self.assertEqual(mig.migrate(True), 1)
+            cp = mig.state.get_checkpoint(record)
+            self.assertEqual(cp["status"], "failed")
+            self.assertNotIn("payload_verified_at", cp)
+            self.assertNotIn("payload_sha256", cp)
+
+    def test_key_redaction(self):
+        secret = "eyJhbGciOiPRIVATEKEYX-Amz-Signature=abc"
+        msg = repack.sanitized_error(RuntimeError("https://x/y?" + secret))
+        self.assertEqual(msg, "RuntimeError")
+        self.assertNotIn(secret, msg)
+
+    def test_old_new_narinfo_sig_compare_allows_db_sigs_empty(self):
+        with tempfile.TemporaryDirectory() as td:
+            mig = repack.Migrator(self.args(td))
+            record = self.record(b"abc")
+            record["sigs"] = []
+            old_info = {**repack.expected_narinfo(record), "Sig": ["hectic:sig"]}
+            new_info = dict(old_info)
+            mig.old_client = mock.Mock(); mig.old_client.get_narinfo.return_value = old_info
+            mig.new_client = mock.Mock(); mig.new_client.get_narinfo.return_value = new_info; mig.new_client.narinfo_url.return_value = "http://127.0.0.1/hectic/abcd.narinfo"
+            mig.new_client.verify_payload.return_value = None
+            self.assertTrue(mig.verify_record(record, False))
+
+    def test_migrate_record_missing_narinfo_raises(self):
+        with tempfile.TemporaryDirectory() as td:
+            mig = repack.Migrator(self.args(td))
+            record = self.record(b"abc")
+            raw = mig.state.raw_path(repack.nar_hash_hex(record["nar_hash"]))
+            raw.write_bytes(b"abc")
+            mig.new_client = mock.Mock(); mig.new_client.get_narinfo.return_value = None
+            with self.assertRaises(repack.RepackError):
+                mig.migrate_record(record)
+
+    def test_migrate_returns_failed_count(self):
+        with tempfile.TemporaryDirectory() as td:
+            mig = repack.Migrator(self.args(td))
+            mig.selected_records = lambda: [self.record(b"abc")]
+            mig.migrate_record = mock.Mock(side_effect=repack.RepackError("new narinfo missing"))
+            self.assertEqual(mig.migrate(False), 1)
+
+    def test_readonly_old_db_rejects_rw_and_missing(self):
+        with self.assertRaises(repack.RepackError):
+            repack.InventoryDB("file:/tmp/x.db?mode=rwc", "hectic")
+        with self.assertRaises(repack.RepackError):
+            repack.InventoryDB("/tmp/definitely-missing-attic.db", "hectic")
+
+    def test_verify_hash_size_fail_closed_unknown_hash(self):
+        with self.assertRaises(repack.RepackError):
+            repack.verify_hash_size(b"x", "sha1:abc", 1, "chunk")
+
+    def args(self, td, old_db=":memory:"):
+        return argparse.Namespace(old_db=old_db, state_dir=str(pathlib.Path(td) / "state"), old_url="http://127.0.0.1:8081", new_url="http://127.0.0.1:8082", host="cache.hectic-lab.com", cache="hectic", atticadm=None, server_config=None, nix="nix", old_storage_endpoint="https://hel1.your-objectstorage.com", old_bucket="cache-hectic-lab", old_region="hel1", workers=2, limit=None, paths_file=None)
+
+    def record(self, data):
+        h = sha(data)
+        return {"cache":"hectic","nar_id":1,"store_path_hash":"abcd","store_path":"/nix/store/abcd-name","references":["/nix/store/ref-ref"],"system":"x86_64-linux","deriver":None,"sigs":["cache:sig"],"ca":None,"nar_hash":"sha256:" + h,"nar_size":len(data)}
+
+
+if __name__ == "__main__":
+    unittest.main()
@@ -0,0 +1,68 @@
+# Gitea runner Nix image
+
+The repo-owned Nix-capable job image is built by the flake package
+`gitea-runner-nix-image`.
+
+```sh
+nix build .#gitea-runner-nix-image
+```
+
+The package emits a Docker archive with the local build tag:
+
+```text
+gitea-runner-nix-image:2026-06-07
+```
+
+That tag is build metadata only. Do not use it as the final Gitea runner label
+mapping because runner job images must be immutable.
+
+## Publication target
+
+Preferred registry:
+
+```text
+gitea.hectic-lab.com/hectic-lab/gitea-runner-nix-image
+```
+
+Publish the archive without adding secrets to the image layers. Controller-owned
+zero-idle runners select this image through `nixImageId` / `GCR_NIX_IMAGE_ID`;
+they do not use a Gitea label-to-container-image mapping:
+
+```text
+nixImageId = "<Hetzner-image-id>";
+```
+
+The `2026-06-07` tag may be pushed as a human-readable companion tag. The
+legacy Kubernetes rollback pool is currently disabled and has no labels.
+If it is restored, its Nix-capable image must be configured separately and
+digest-pinned before enabling a `nix` label.
+
+Do not use a tag-only image for a restored Kubernetes rollback pool.
+
+## Task 7 publication status
+
+Local build evidence is recorded in
+`.sisyphus/evidence/task-7-image-digest.txt`. Kubernetes pull smoke is recorded in
+`.sisyphus/evidence/task-7-image-pull.txt` and is blocked here because `kubectl`
+is not installed or not on `PATH`.
+
+After importing the archive as a Hetzner image, record its image ID in the
+controller host configuration before dispatching Nix jobs.
+
+## Image contents
+
+The image includes `nix`, `git`, `bash`, `coreutils`, and `cacert`. Its
+`/etc/nix/nix.conf` enables flakes and configures the repo substituters from the
+top-level `flake.nix`:
+
+```text
+experimental-features = nix-command flakes
+substituters = https://cache.nixos.org https://cache.hectic-lab.com/hectic
+http2 = false
+trusted-public-keys = cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY= hectic:KMQsKow4SoA9K2vOJlOljmx7/Zpf91Yy+5qEtxDDCzA=
+sandbox = false
+```
+
+No Gitea runner token, SSH key, SOPS key, kubeconfig, Hetzner token, or S3
+credential belongs in this image. Runtime secrets stay with the Kubernetes
+runner configuration and token-file mount contract.
@@ -0,0 +1,154 @@
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: gitea-runner-lifecycle
+  namespace: gitea-runners
+  labels:
+    app.kubernetes.io/name: gitea-runner-lifecycle
+    app.kubernetes.io/part-of: gitea-actions
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  name: gitea-runner-lifecycle
+  namespace: gitea-runners
+  labels:
+    app.kubernetes.io/name: gitea-runner-lifecycle
+    app.kubernetes.io/part-of: gitea-actions
+rules:
+  - apiGroups:
+      - ""
+    resources:
+      - pods
+      - persistentvolumeclaims
+    verbs:
+      - get
+      - list
+  - apiGroups:
+      - apps
+    resources:
+      - statefulsets
+    verbs:
+      - get
+      - list
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: gitea-runner-lifecycle
+  namespace: gitea-runners
+  labels:
+    app.kubernetes.io/name: gitea-runner-lifecycle
+    app.kubernetes.io/part-of: gitea-actions
+subjects:
+  - kind: ServiceAccount
+    name: gitea-runner-lifecycle
+    namespace: gitea-runners
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: gitea-runner-lifecycle
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: gitea-runner-lifecycle
+  namespace: gitea-runners
+  labels:
+    app.kubernetes.io/name: gitea-runner-lifecycle
+    app.kubernetes.io/part-of: gitea-actions
+data:
+  cleanup-dry-run.sh: |
+    #!/bin/sh
+    set -eu
+
+    namespace="${RUNNER_NAMESPACE:-gitea-runners}"
+    mode="${CLEANUP_MODE:-dry-run}"
+    selector="app.kubernetes.io/name=gitea-runner"
+
+    if [ "$namespace" != "gitea-runners" ]; then
+      printf 'refusing to run outside namespace gitea-runners: %s\n' "$namespace" >&2
+      exit 13
+    fi
+
+    if [ "$mode" != "dry-run" ]; then
+      printf 'refusing destructive mode: set CLEANUP_MODE=dry-run for this CronJob\n' >&2
+      exit 13
+    fi
+
+    printf 'gitea runner lifecycle cleanup dry-run\n'
+    printf 'namespace: %s\n' "$namespace"
+    printf 'mode: %s\n\n' "$mode"
+
+    printf 'StatefulSet:\n'
+    kubectl -n "$namespace" get statefulset gitea-runner -o wide
+
+    printf '\nActive runner pods:\n'
+    kubectl -n "$namespace" get pods -l "$selector" -o wide
+
+    printf '\nRunner /data PVCs:\n'
+    kubectl -n "$namespace" get pvc -l "$selector" -o wide
+
+    printf '\nPVCs whose matching StatefulSet pod is absent (candidates only; no deletion):\n'
+    found_candidate=0
+    for pvc in $(kubectl -n "$namespace" get pvc -l "$selector" -o jsonpath='{range .items[*]}{.metadata.name}{"\n"}{end}'); do
+      pod="${pvc#data-}"
+      if ! kubectl -n "$namespace" get pod "$pod" >/dev/null 2>&1; then
+        found_candidate=1
+        printf 'candidate pvc=%s expected_pod=%s action=investigate-before-delete\n' "$pvc" "$pod"
+      fi
+    done
+
+    if [ "$found_candidate" -eq 0 ]; then
+      printf 'none\n'
+    fi
+
+    printf '\nGitea registration reconciliation:\n'
+    printf 'dry-run only: compare the pod/PVC list above with Gitea org runner registrations.\n'
+    printf 'only deregister a runner after its pod/PVC was intentionally deleted or /data/.runner was intentionally reset.\n'
+---
+apiVersion: batch/v1
+kind: CronJob
+metadata:
+  name: gitea-runner-cleanup-dry-run
+  namespace: gitea-runners
+  labels:
+    app.kubernetes.io/name: gitea-runner-lifecycle
+    app.kubernetes.io/part-of: gitea-actions
+spec:
+  schedule: "17 3 * * *"
+  concurrencyPolicy: Forbid
+  successfulJobsHistoryLimit: 3
+  failedJobsHistoryLimit: 3
+  jobTemplate:
+    spec:
+      ttlSecondsAfterFinished: 3600
+      template:
+        metadata:
+          labels:
+            app.kubernetes.io/name: gitea-runner-lifecycle
+            app.kubernetes.io/part-of: gitea-actions
+        spec:
+          serviceAccountName: gitea-runner-lifecycle
+          restartPolicy: Never
+          containers:
+            - name: cleanup-dry-run
+              image: bitnami/kubectl:1.30
+              imagePullPolicy: IfNotPresent
+              command:
+                - /bin/sh
+                - /scripts/cleanup-dry-run.sh
+              env:
+                - name: RUNNER_NAMESPACE
+                  value: gitea-runners
+                - name: CLEANUP_MODE
+                  value: dry-run
+              volumeMounts:
+                - name: lifecycle-scripts
+                  mountPath: /scripts
+                  readOnly: true
+          volumes:
+            - name: lifecycle-scripts
+              configMap:
+                name: gitea-runner-lifecycle
+                defaultMode: 0555
@@ -0,0 +1,9 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+  - namespace.yaml
+  - service.yaml
+  - service-account.yaml
+  - runner-config.yaml
+  - statefulset.yaml
+  - cleanup-lifecycle.yaml
@@ -0,0 +1,7 @@
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: gitea-runners
+  labels:
+    app.kubernetes.io/name: gitea-runner
+    app.kubernetes.io/part-of: gitea-actions
@@ -0,0 +1,34 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: gitea-runner-config
+  namespace: gitea-runners
+  labels:
+    app.kubernetes.io/name: gitea-runner
+    app.kubernetes.io/part-of: gitea-actions
+data:
+  config.yaml: |
+    log:
+      level: info
+
+    runner:
+      file: /data/.runner
+      capacity: 1
+      envs: {}
+      timeout: 3h
+      insecure: false
+      fetch_timeout: 5s
+      fetch_interval: 2s
+      # Persistent pool is disabled; controller owns all runner labels.
+      labels: []
+
+    cache:
+      enabled: true
+      dir: /data/cache
+
+    container:
+      network: bridge
+      privileged: false
+      force_pull: true
+      valid_volumes: []
+      docker_host: unix:///runner-docker/docker.sock
@@ -0,0 +1,36 @@
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: gitea-runner
+  namespace: gitea-runners
+  labels:
+    app.kubernetes.io/name: gitea-runner
+    app.kubernetes.io/part-of: gitea-actions
+automountServiceAccountToken: false
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  name: gitea-runner
+  namespace: gitea-runners
+  labels:
+    app.kubernetes.io/name: gitea-runner
+    app.kubernetes.io/part-of: gitea-actions
+rules: []
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: gitea-runner
+  namespace: gitea-runners
+  labels:
+    app.kubernetes.io/name: gitea-runner
+    app.kubernetes.io/part-of: gitea-actions
+subjects:
+  - kind: ServiceAccount
+    name: gitea-runner
+    namespace: gitea-runners
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: gitea-runner
@@ -0,0 +1,16 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: gitea-runner
+  namespace: gitea-runners
+  labels:
+    app.kubernetes.io/name: gitea-runner
+    app.kubernetes.io/part-of: gitea-actions
+spec:
+  clusterIP: None
+  selector:
+    app.kubernetes.io/name: gitea-runner
+  ports:
+    - name: cache
+      port: 8088
+      targetPort: cache
@@ -0,0 +1,157 @@
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+  name: gitea-runner
+  namespace: gitea-runners
+  labels:
+    app.kubernetes.io/name: gitea-runner
+    app.kubernetes.io/part-of: gitea-actions
+spec:
+  serviceName: gitea-runner
+  # Rollback-only pool. Keep manifest present, default replicas 0.
+  replicas: 0
+  podManagementPolicy: Parallel
+  selector:
+    matchLabels:
+      app.kubernetes.io/name: gitea-runner
+  template:
+    metadata:
+      labels:
+        app.kubernetes.io/name: gitea-runner
+        app.kubernetes.io/part-of: gitea-actions
+      annotations:
+        hectic-lab.com/security-note: "Privileged rootful DinD is limited to trusted internal Gitea workflows only. Do not enable untrusted fork or PR jobs for this pool."
+    spec:
+      serviceAccountName: gitea-runner
+      automountServiceAccountToken: false
+      terminationGracePeriodSeconds: 60
+      securityContext:
+        fsGroup: 1000
+      containers:
+        - name: runner
+          image: gitea/act_runner:0.2.11
+          imagePullPolicy: IfNotPresent
+          env:
+            - name: GITEA_INSTANCE_URL
+              value: https://gitea.hectic-lab.com
+            - name: GITEA_RUNNER_REGISTRATION_TOKEN_FILE
+              value: /runner-secrets/token
+            - name: CONFIG_FILE
+              value: /runner-config/config.yaml
+            - name: DOCKER_HOST
+              value: unix:///runner-docker/docker.sock
+          ports:
+            - name: cache
+              containerPort: 8088
+          resources:
+            requests:
+              cpu: 250m
+              memory: 256Mi
+            limits:
+              cpu: "1"
+              memory: 1Gi
+          livenessProbe:
+            exec:
+              command:
+                - /bin/sh
+                - -ec
+                - test -s /data/.runner && test -S /runner-docker/docker.sock
+            initialDelaySeconds: 60
+            periodSeconds: 30
+            timeoutSeconds: 5
+            failureThreshold: 6
+          readinessProbe:
+            exec:
+              command:
+                - /bin/sh
+                - -ec
+                - test -S /runner-docker/docker.sock
+            initialDelaySeconds: 15
+            periodSeconds: 10
+            timeoutSeconds: 5
+            failureThreshold: 6
+          volumeMounts:
+            - name: data
+              mountPath: /data
+            - name: config
+              mountPath: /runner-config
+              readOnly: true
+            - name: runner-token
+              mountPath: /runner-secrets
+              readOnly: true
+            - name: docker-socket
+              mountPath: /runner-docker
+        - name: docker
+          image: docker:27-dind
+          imagePullPolicy: IfNotPresent
+          args:
+            - --host=unix:///runner-docker/docker.sock
+            - --storage-driver=overlay2
+            - --tls=false
+          env:
+            - name: DOCKER_TLS_CERTDIR
+              value: ""
+            - name: DOCKER_HOST
+              value: unix:///runner-docker/docker.sock
+          securityContext:
+            # Privileged rootful DinD is intentionally scoped to this trusted
+            # internal runner pool; never expose it to untrusted fork/PR jobs.
+            privileged: true
+          resources:
+            requests:
+              cpu: 500m
+              memory: 1Gi
+            limits:
+              cpu: "2"
+              memory: 4Gi
+          livenessProbe:
+            exec:
+              command:
+                - docker
+                - info
+            initialDelaySeconds: 60
+            periodSeconds: 30
+            timeoutSeconds: 10
+            failureThreshold: 6
+          readinessProbe:
+            exec:
+              command:
+                - docker
+                - info
+            initialDelaySeconds: 20
+            periodSeconds: 10
+            timeoutSeconds: 10
+            failureThreshold: 6
+          volumeMounts:
+            - name: docker-socket
+              mountPath: /runner-docker
+            - name: docker-graph
+              mountPath: /var/lib/docker
+      volumes:
+        - name: config
+          configMap:
+            name: gitea-runner-config
+        - name: runner-token
+          secret:
+            secretName: gitea-runner-token
+            items:
+              - key: token
+                path: token
+            defaultMode: 0400
+        - name: docker-socket
+          emptyDir: {}
+        - name: docker-graph
+          emptyDir: {}
+  volumeClaimTemplates:
+    - metadata:
+        name: data
+        labels:
+          app.kubernetes.io/name: gitea-runner
+          app.kubernetes.io/part-of: gitea-actions
+      spec:
+        accessModes:
+          - ReadWriteOnce
+        storageClassName: hcloud-volumes
+        resources:
+          requests:
+            storage: 20Gi
@@ -0,0 +1,29 @@
+# OpenTofu working directory and downloaded modules/providers.
+.terraform/
+.terraform.lock.hcl
+
+# State must live in the S3 backend for production. Local state is allowed only
+# for throwaway syntax checks with `tofu init -backend=false` and must not be
+# committed.
+terraform.tfstate
+terraform.tfstate.*
+*.tfstate
+*.tfstate.*
+crash.log
+crash.*.log
+
+# Plans can contain secrets or derived infrastructure data.
+*.tfplan
+*.plan
+kubeconfig
+kubeconfig.yaml
+*_kubeconfig.yaml
+
+# Variable files commonly carry credentials. Keep production inputs in SOPS or
+# external environment/configuration, not in checked-in files.
+*.tfvars
+*.tfvars.json
+override.tf
+override.tf.json
+*_override.tf
+*_override.tf.json
@@ -0,0 +1,96 @@
+# Gitea runner OpenTofu backend contract
+
+This directory defines the safe backend, provider contract, and kube-hetzner
+cluster stack for the Gitea runner Kubernetes cluster.
+
+## Required backend
+
+Production state must use the OpenTofu S3 backend in `backend.tf`:
+
+- bucket: `gitea-runner-hectic-lab`
+- key: `gitea-runners/kube-hetzner/terraform.tfstate`
+- region: `fsn1`, aligned with the target Hetzner location
+- encryption: `encrypt = true`
+- locking: `use_lockfile = true` where the selected S3-compatible endpoint
+  supports it
+
+Before any production `tofu init`, verify the S3-compatible endpoint, credential
+source, bucket versioning, encryption behavior, and lockfile support for the
+chosen object-storage provider. Keep backend authentication externalized through
+environment variables, AWS-compatible shared config, or the production secret
+injection path from Task 3. Do not add `access_key`, `secret_key`, Hetzner
+tokens, runner tokens, kubeconfig material, or decrypted SOPS data to checked-in
+OpenTofu files.
+
+## Local state safety
+
+Production local state is forbidden. Only syntax-only validation/prototyping may
+use local state, and it must use backend-disabled initialization:
+
+```sh
+tofu -chdir=infra/gitea-runners/opentofu init -backend=false
+tofu -chdir=infra/gitea-runners/opentofu validate
+```
+
+Fail the run if production local state appears:
+
+```sh
+test ! -e infra/gitea-runners/opentofu/terraform.tfstate
+test ! -e infra/gitea-runners/opentofu/terraform.tfstate.backup
+grep -R 'backend "s3"' infra/gitea-runners/opentofu
+```
+
+The `.gitignore` in this directory blocks local state, plans, downloaded
+providers/modules, and variable files from being committed. Treat any local
+state file as disposable validation residue, never as production state.
+
+## Provider and module pins
+
+`versions.tf` pins the OpenTofu-compatible Hetzner Cloud provider to
+`hetznercloud/hcloud` version `1.60.1`. kube-hetzner research for this plan
+observed module version `2.19.3`, source `kube-hetzner/kube-hetzner/hcloud`, and
+module minimum hcloud provider requirement `>= 1.59.0`; these values are recorded
+as locals so Task 5 can wire the module without re-opening the version contract.
+
+`providers.tf` leaves the `hcloud` provider empty so authentication comes from
+the provider's external environment/config mechanisms such as `HCLOUD_TOKEN`.
+Do not set token values in `.tf` or `.tfvars` files.
+
+## Cluster shape
+
+The default cluster is deliberately fixed-size:
+
+- cluster name: `gitea-runners`
+- Hetzner location: `fsn1`
+- private network region: `eu-central`
+- control plane: one `cpx22` node in pool `control-plane`
+- workers: one `cpx22` node in pool `runner-workers`
+- storage: Hetzner CSI enabled with expected StorageClass `hcloud-volumes`
+- Longhorn: disabled
+- autoscaling/KEDA: not enabled in this stack
+
+The default baseline uses one `cpx22` worker to keep the idle bill low while
+still supporting trusted internal jobs. To scale toward higher concurrency
+later, keep autoscaling disabled and either raise `worker_count` or increase
+`worker_server_type`, then run a fresh `tofu plan` and the Task 11 Kubernetes
+pressure checks before applying.
+
+Required inputs must come from environment or secret injection, for example
+`TF_VAR_hcloud_token`, `TF_VAR_ssh_public_key`, and `TF_VAR_ssh_private_key`.
+Set `TF_VAR_firewall_ssh_source` and `TF_VAR_firewall_kube_api_source` explicitly
+to trusted CIDR ranges before planning; the cluster must not expose SSH or port
+6443 to `0.0.0.0/0`.
+If the public key is already registered in Hetzner, set
+`TF_VAR_hcloud_ssh_key_id` to reuse it instead of creating a duplicate key.
+Do not commit `.tfvars` files. kube-hetzner v2.19.3 writes the generated
+kubeconfig to `./<cluster_name>_kubeconfig.yaml` when `create_kubeconfig` is
+enabled; this path is ignored as operational secret material.
+
+## Known state caveat
+
+kube-hetzner may thread `hcloud_token` into Kubernetes secrets/state through its
+internal `kube_system_secrets` handling. This task does not claim that risk is
+solved. Task 5 must verify the generated plan and state before production apply
+and prove that Hetzner tokens, S3 credentials, runner tokens, kubeconfig private
+keys, and decrypted secrets are absent from committed files and unsafe state
+evidence.
@@ -0,0 +1,27 @@
+terraform {
+  backend "s3" {
+    bucket = "gitea-runner-hectic-lab"
+    key    = "gitea-runners/kube-hetzner/terraform.tfstate"
+    region = "hel1"
+
+    endpoints = {
+      s3 = "https://hel1.your-objectstorage.com"
+    }
+
+    skip_credentials_validation = true
+    skip_metadata_api_check     = true
+    skip_region_validation      = true
+    skip_requesting_account_id  = true
+    use_path_style              = true
+    encrypt                     = false
+    skip_s3_checksum            = true
+    use_lockfile                = true
+  }
+}
+
+check "remote_state_contract" {
+  assert {
+    condition     = local.production_remote_state
+    error_message = "Production OpenTofu state must use the configured S3 backend; local production state is forbidden."
+  }
+}
@@ -0,0 +1,39 @@
+"apiVersion": "kustomize.config.k8s.io/v1beta1"
+"kind": "Kustomization"
+"patches":
+- "patch": |
+    apiVersion: apps/v1
+    kind: Deployment
+    metadata:
+      name: system-upgrade-controller
+      namespace: system-upgrade
+    spec:
+      template:
+        spec:
+          containers:
+            - name: system-upgrade-controller
+              volumeMounts:
+                - name: ca-certificates
+                  mountPath: /var/lib/ca-certificates
+          volumes:
+            - name: ca-certificates
+              hostPath:
+                path: /var/lib/ca-certificates
+                type: Directory
+  "target":
+    "group": "apps"
+    "kind": "Deployment"
+    "name": "system-upgrade-controller"
+    "namespace": "system-upgrade"
+    "version": "v1"
+- "path": "kured.yaml"
+- "path": "ccm.yaml"
+"resources":
+- "https://github.com/kubereboot/kured/releases/download/1.23.0/kured-1.23.0-combined.yaml"
+- "https://github.com/rancher/system-upgrade-controller/releases/download/v0.18.0/system-upgrade-controller.yaml"
+- "https://github.com/rancher/system-upgrade-controller/releases/download/v0.18.0/crd.yaml"
+- "https://github.com/hetznercloud/hcloud-cloud-controller-manager/releases/download/v1.35.0/ccm-networks.yaml"
+- "hcloud-csi.yaml"
+- "traefik_ingress.yaml"
+- "flannel-rbac.yaml"
+- "cert_manager.yaml"
@@ -0,0 +1,63 @@
+locals {
+  default_storage_class = "hcloud-volumes"
+
+  control_plane_nodepools = [
+    {
+      name        = "control-plane"
+      server_type = var.control_plane_server_type
+      location    = var.hetzner_location
+      labels      = []
+      taints      = []
+      count       = 1
+    },
+  ]
+
+  agent_nodepools = [
+    {
+      name        = "runner-workers"
+      server_type = var.worker_server_type
+      location    = var.hetzner_location
+      labels      = ["node-role.hectic-lab/gitea-runner=true"]
+      taints      = []
+      count       = var.worker_count
+    },
+  ]
+}
+
+module "kube_hetzner" {
+  source  = "kube-hetzner/kube-hetzner/hcloud"
+  version = "2.19.3"
+
+  providers = {
+    hcloud = hcloud
+  }
+
+  hcloud_token    = var.hcloud_token
+  ssh_public_key  = var.ssh_public_key
+  ssh_private_key = var.ssh_private_key
+
+  cluster_name = var.cluster_name
+  base_domain  = var.base_domain
+
+  # kube-hetzner v2.19.3 writes <cluster_name>_kubeconfig.yaml; outputs below
+  # expose that expected path without outputting kubeconfig private key material.
+  create_kubeconfig = true
+
+  network_region          = var.network_region
+  load_balancer_location  = var.hetzner_location
+  control_plane_nodepools = local.control_plane_nodepools
+  agent_nodepools         = local.agent_nodepools
+  firewall_ssh_source     = var.firewall_ssh_source
+  firewall_kube_api_source = var.firewall_kube_api_source
+  hcloud_ssh_key_id       = var.hcloud_ssh_key_id
+
+  # Hetzner CSI is the required StorageClass provider for runner PVCs.
+  disable_hetzner_csi = false
+
+  # Longhorn is intentionally off; the initial runner PVCs use Hetzner CSI only.
+  enable_longhorn = false
+
+  # Scaling note: for 10 trusted DinD jobs later, keep autoscaling disabled and
+  # raise worker_count to 5 or increase worker_server_type after validating pod
+  # CPU, memory, and ephemeral-storage pressure in Task 11.
+}
@@ -0,0 +1,22 @@
+output "kubeconfig_path" {
+  description = "Path where kube-hetzner writes kubeconfig after apply. The file is operational secret material and must not be committed."
+  value       = coalesce(var.kubeconfig_path, "./${var.cluster_name}_kubeconfig.yaml")
+}
+
+output "cluster_name" {
+  description = "kube-hetzner cluster name."
+  value       = var.cluster_name
+}
+
+output "node_pool_names" {
+  description = "Control-plane and worker node pool names used by this stack."
+  value = {
+    control_plane = [for pool in local.control_plane_nodepools : pool.name]
+    workers       = [for pool in local.agent_nodepools : pool.name]
+  }
+}
+
+output "default_storage_class" {
+  description = "Default Hetzner CSI StorageClass expected for runner PVCs."
+  value       = local.default_storage_class
+}
@@ -0,0 +1 @@
+provider "hcloud" {}
@@ -0,0 +1,90 @@
+variable "hcloud_token" {
+  description = "Hetzner Cloud API token for kube-hetzner. Set with TF_VAR_hcloud_token or secret injection only; never commit it. kube-hetzner may place this value into Kubernetes secret resources/state, so scan plans before apply."
+  type        = string
+  sensitive   = true
+}
+
+variable "ssh_public_key" {
+  description = "SSH public key installed on cluster nodes. Supply from an external file or secret injection path."
+  type        = string
+}
+
+variable "ssh_private_key" {
+  description = "SSH private key used by kube-hetzner during bootstrap. Supply from an external file or secret injection path; never commit it."
+  type        = string
+  sensitive   = true
+}
+
+variable "cluster_name" {
+  description = "Name for the kube-hetzner runner cluster."
+  type        = string
+  default     = "gitea-runners"
+
+  validation {
+    condition     = can(regex("^[a-z0-9-]+$", var.cluster_name))
+    error_message = "cluster_name must contain only lowercase letters, numbers, and dashes."
+  }
+}
+
+variable "hetzner_location" {
+  description = "Hetzner Cloud location for all node pools. fsn1 keeps the first runner cluster in Falkenstein."
+  type        = string
+  default     = "fsn1"
+}
+
+variable "network_region" {
+  description = "Hetzner private network region. eu-central covers fsn1."
+  type        = string
+  default     = "eu-central"
+}
+
+variable "control_plane_server_type" {
+  description = "Default control-plane server type. cpx22 is small but leaves headroom for kube-system workloads."
+  type        = string
+  default     = "cpx22"
+}
+
+variable "worker_server_type" {
+  description = "Default worker server type for the budget trusted DinD cluster. One cpx22 worker keeps the cluster baseline cheap; scale out later if rollback capacity requires it."
+  type        = string
+  default     = "cpx22"
+}
+
+variable "worker_count" {
+  description = "Fixed worker count. Default is a single cheap worker for low-cost baseline operation; increase later if concurrency requires it."
+  type        = number
+  default     = 1
+
+  validation {
+    condition     = var.worker_count >= 1
+    error_message = "worker_count must be at least 1."
+  }
+}
+
+variable "kubeconfig_path" {
+  description = "Expected kubeconfig path. kube-hetzner v2.19.3 writes this as <cluster_name>_kubeconfig.yaml when create_kubeconfig is true."
+  type        = string
+  default     = null
+}
+
+variable "base_domain" {
+  description = "Optional base domain for node reverse DNS. Empty keeps kube-hetzner defaults."
+  type        = string
+  default     = ""
+}
+
+variable "firewall_ssh_source" {
+  description = "CIDR ranges allowed to SSH to cluster nodes. Set explicitly; never expose SSH to the public internet."
+  type        = list(string)
+}
+
+variable "firewall_kube_api_source" {
+  description = "CIDR ranges allowed to reach the Kubernetes API. Set explicitly; never expose port 6443 to the public internet."
+  type        = list(string)
+}
+
+variable "hcloud_ssh_key_id" {
+  description = "Existing Hetzner SSH key ID. Set this when the public key is already registered in the project."
+  type        = string
+  default     = null
+}
@@ -0,0 +1,17 @@
+terraform {
+  required_version = ">= 1.10.1"
+
+  required_providers {
+    hcloud = {
+      source  = "hetznercloud/hcloud"
+      version = "1.60.1"
+    }
+  }
+}
+
+locals {
+  kube_hetzner_module_source  = "kube-hetzner/kube-hetzner/hcloud"
+  kube_hetzner_module_version = "2.19.3"
+  hcloud_provider_minimum     = ">= 1.59.0"
+  production_remote_state     = true
+}
@@ -0,0 +1,667 @@
+# Gitea Runner Infrastructure Runbook
+
+## Scope
+
+This directory is the repo-owned boundary for the Gitea Actions runner pool.
+The controller is the active on-demand path; Kubernetes manifests and the
+Nix-capable image are retained for manual rollback and maintenance.
+
+The target service is `https://gitea.hectic-lab.com` for the Gitea organization
+`hectic-lab`. The pool is trusted-only. "Ephemeral" means each controller VM
+and workflow job is disposable; the Kubernetes StatefulSet is rollback-only.
+
+## Immutable decisions
+
+- Infrastructure is managed with OpenTofu command examples only, using the
+  `tofu` CLI.
+- Cloud provider is Hetzner; cluster bootstrap uses kube-hetzner.
+- Remote state uses the S3 backend bucket `gitea-runner-hectic-lab`.
+- Runner implementation is the non-Enterprise `gitea/runner`.
+- Runner registration uses a Gitea organization-scoped token for `hectic-lab`.
+- Runtime token delivery is SOPS-backed and mounted into the runner pod as a
+  file read through `GITEA_RUNNER_REGISTRATION_TOKEN_FILE`; plaintext token
+  environment variables are not the contract.
+- Kubernetes runner lifecycle uses a StatefulSet with one PVC per pod for
+  `/data`, including `/data/.runner`.
+- Container builds run through privileged rootful DinD inside trusted runner
+  pods; host Docker socket mounting is not an implementation path.
+- `ubuntu-latest` and `nix` are controller-managed on-demand aliases for
+  `gross-x86` and `gross-nix-x86`; the Kubernetes pool has no active labels.
+- First scope is trusted internal workflows only, with no untrusted fork or PR
+  workflow support.
+- On-demand allocation is handled by the repo-owned controller; Kubernetes is
+  not an active autoscaling path.
+
+## Lifecycle boundaries
+
+- `infra/gitea-runners/opentofu/`: OpenTofu stack for the S3 backend
+  contract, Hetzner provider configuration, and kube-hetzner module wiring.
+- `infra/gitea-runners/k8s/`: rollback-only namespace, ConfigMap, Secret mount,
+  StatefulSet, PVC, DinD sidecar, cleanup, and operational manifest work.
+- `infra/gitea-runners/image/`: notes and handoff for the optional Kubernetes
+  rollback image; active on-demand Nix image is selected by Hetzner image ID.
+- `infra/gitea-runners/runbook.md`: this contract plus later operational
+  commands, rollback notes, and acceptance evidence references.
+
+## Guardrails
+
+- Enterprise ARC/actions-runner-controller are rejected alternatives and must
+  not be implemented here. Do not add ARC custom resources, controller install
+  instructions, or GitHub Actions ARC assumptions.
+- Untrusted fork/PR workflows are out of first scope; privileged DinD is only
+  acceptable for trusted internal jobs.
+- The persistent StatefulSet is rollback-only and defaults to zero replicas;
+  normal jobs use controller-managed on-demand VMs.
+- No actual secrets are committed: no kubeconfig, runner token, Hetzner token,
+  S3 credentials, decrypted SOPS files, or SOPS age keys.
+- OpenTofu must not manage plaintext Kubernetes Secrets containing the Gitea
+  runner token; Kubernetes receives the token as a mounted file secret instead.
+- Do not use `localhost` or `127.0.0.1` as the Gitea URL inside job containers;
+  jobs must reach the public HTTPS service.
+
+## Initial acceptance commands
+
+Run from the repository root:
+
+```sh
+test -d infra/gitea-runners/opentofu && test -d infra/gitea-runners/k8s && test -d infra/gitea-runners/image
+test -f infra/gitea-runners/runbook.md
+grep -n "OpenTofu\|kube-hetzner\|StatefulSet\|DinD\|SOPS\|trusted" infra/gitea-runners/runbook.md
+grep -R "[E]nterprise ARC\|[a]ctions-runner-controller" infra/gitea-runners
+grep -R "[t]erraform " infra/gitea-runners || true
+grep -R "[D]ECISION NEEDED" infra/gitea-runners || true
+```
+
+Expected outcomes: the directory and file checks exit 0; the architecture-term
+grep shows this contract; ARC references appear only in the rejected-alternative
+guardrail above; there are no forbidden CLI command examples and no unresolved
+decision placeholders.
+
+## Downstream placeholders
+
+- Task 2: add OpenTofu backend/provider files and verify S3 state safety.
+- Task 3: add SOPS secret contract and runtime token delivery details.
+- Task 4: define or package the Nix-capable runner image for the `nix` label.
+- Task 5+: provision kube-hetzner, add Kubernetes resources, verify workflows,
+  and document cleanup, rollback, and scaling operations.
+
+## Runner lifecycle cleanup
+
+All lifecycle commands are scoped to the runner namespace:
+
+```sh
+kubectl -n gitea-runners get statefulset gitea-runner
+kubectl -n gitea-runners get pods -l app.kubernetes.io/name=gitea-runner -o wide
+kubectl -n gitea-runners get pvc -l app.kubernetes.io/name=gitea-runner -o wide
+```
+
+The scheduled cleanup manifest is dry-run only. It lists the StatefulSet, active
+runner pods, runner PVCs, and PVCs whose expected StatefulSet pod is absent. It
+does not delete pods, PVCs, Docker data, or Gitea runner registrations.
+
+Run the same inventory on demand without waiting for the schedule:
+
+```sh
+kubectl -n gitea-runners create job gitea-runner-cleanup-dry-run-manual --from=cronjob/gitea-runner-cleanup-dry-run
+kubectl -n gitea-runners wait --for=condition=complete job/gitea-runner-cleanup-dry-run-manual --timeout=2m
+kubectl -n gitea-runners logs job/gitea-runner-cleanup-dry-run-manual -c cleanup-dry-run
+```
+
+The cleanup job template has `ttlSecondsAfterFinished: 3600`, so completed
+manual dry-run jobs are garbage-collected by Kubernetes instead of requiring an
+operator to remove finished jobs manually.
+
+If a PVC such as `data-gitea-runner-3` is intentionally deleted, the matching
+pod loses `/data/.runner`. That runner identity must then be deregistered from
+Gitea or the replacement pod must be allowed to re-register intentionally with
+the current organization runner token. Do not delete an active runner PVC as a
+normal cleanup step.
+
+Non-UI Gitea registration reconciliation uses the Gitea API with a separate
+admin token. Store that token outside this repository and pass it as a file; do
+not print it:
+
+```sh
+kubectl -n gitea-runners create secret generic gitea-runner-admin-token --from-file=token=/secure/path/gitea-admin-token
+kubectl -n gitea-runners run gitea-runner-registration-dry-run \
+  --restart=Never \
+  --image=curlimages/curl:8.10.1 \
+  --overrides='{"spec":{"containers":[{"name":"gitea-runner-registration-dry-run","image":"curlimages/curl:8.10.1","command":["/bin/sh","-ec","umask 077; cfg=$(mktemp); trap '\''rm -f \"$cfg\"'\'' EXIT; { printf '\''header = \"Authorization: token '\''; cat /admin-token/token; printf '\''\"\\n'\''; printf '\''url = \"https://gitea.hectic-lab.com/api/v1/orgs/hectic-lab/actions/runners\"\\n'\''; } > \"$cfg\"; curl -fsS --config \"$cfg\""],"volumeMounts":[{"name":"admin-token","mountPath":"/admin-token","readOnly":true}]}],"volumes":[{"name":"admin-token","secret":{"secretName":"gitea-runner-admin-token","defaultMode":256}}]}}'
+kubectl -n gitea-runners logs pod/gitea-runner-registration-dry-run
+```
+
+Delete the temporary `gitea-runner-admin-token` Secret only after the dry-run pod
+has completed and its logs have been collected. Do not keep this admin token in
+the runner namespace longer than the reconciliation window.
+
+Only remove a stale Gitea runner registration after the corresponding pod/PVC
+was intentionally deleted or `/data/.runner` was intentionally reset. Prefer a
+Gitea CLI/API deletion from the Gitea server or an admin workstation; manual UI
+cleanup is a fallback, not the only path. Record the removed runner name and the
+Kubernetes PVC/pod deletion that made it stale.
+
+After the dry-run list identifies a stale registration and the PVC/pod deletion
+has been recorded, remove that exact Gitea runner by id through the API:
+
+```sh
+runner_id='REPLACE_WITH_STALE_RUNNER_ID'
+umask 077
+curl_config=$(mktemp /tmp/gitea-runner-admin-curl.XXXXXX)
+trap 'rm -f "$curl_config"' EXIT
+{
+  printf 'request = "DELETE"\n'
+  printf 'header = "Authorization: token '
+  cat /secure/path/gitea-admin-token
+  printf '"\n'
+  printf 'url = "https://gitea.hectic-lab.com/api/v1/orgs/hectic-lab/actions/runners/%s"\n' "$runner_id"
+} > "$curl_config"
+curl -fsS --config "$curl_config"
+```
+
+Do not run the delete command for a runner that still has an active
+`gitea-runner-*` pod or a retained `data-gitea-runner-*` PVC unless that PVC is
+being intentionally reset for re-registration.
+
+## Docker-in-Docker storage cleanup
+
+Docker layers live inside each DinD sidecar at `/var/lib/docker`, backed by the
+pod-local `docker-graph` `emptyDir`; the host Docker socket is not used. Always
+list disk usage before pruning, and run the command only against the `docker`
+container in runner pods in `gitea-runners`. Because this storage is pod-local,
+loop over pods for pool-wide cleanup:
+
+```sh
+for pod in $(kubectl -n gitea-runners get pods -l app.kubernetes.io/name=gitea-runner -o jsonpath='{range .items[*]}{.metadata.name}{"\n"}{end}'); do
+  kubectl -n gitea-runners exec "pod/${pod}" -c docker -- docker system df
+done
+
+for pod in $(kubectl -n gitea-runners get pods -l app.kubernetes.io/name=gitea-runner -o jsonpath='{range .items[*]}{.metadata.name}{"\n"}{end}'); do
+  kubectl -n gitea-runners exec "pod/${pod}" -c docker -- docker system prune --all --force --filter until=24h
+done
+
+for pod in $(kubectl -n gitea-runners get pods -l app.kubernetes.io/name=gitea-runner -o jsonpath='{range .items[*]}{.metadata.name}{"\n"}{end}'); do
+  kubectl -n gitea-runners exec "pod/${pod}" -c docker -- docker system df
+done
+```
+
+For one pod, replace the StatefulSet target with the pod name:
+
+```sh
+kubectl -n gitea-runners exec pod/gitea-runner-0 -c docker -- docker system df
+kubectl -n gitea-runners exec pod/gitea-runner-0 -c docker -- docker system prune --all --force --filter until=24h
+```
+
+Do not run host-level Docker cleanup commands and do not mount or prune a host
+Docker socket. If a pod is deleted, its `emptyDir` Docker graph is removed by
+Kubernetes; the `/data` PVC remains and still controls runner identity.
+
+## Token rotation
+
+Rotate the Gitea organization runner token without printing decrypted values:
+
+```sh
+sops sus/gitea-runners.yaml
+umask 077
+token_file=$(mktemp /tmp/gitea-runner-token.XXXXXX)
+trap 'rm -f "$token_file"' EXIT
+sops -d --extract '["gitea"]["hectic-lab"]["org-runner-registration-token"]' sus/gitea-runners.yaml > "$token_file"
+kubectl -n gitea-runners create secret generic gitea-runner-token \
+  --from-file=token="$token_file" \
+  --dry-run=client \
+  -o yaml | kubectl -n gitea-runners apply -f -
+kubectl -n gitea-runners rollout restart statefulset/gitea-runner
+kubectl -n gitea-runners rollout status statefulset/gitea-runner --timeout=10m
+kubectl -n gitea-runners get pods -l app.kubernetes.io/name=gitea-runner -o wide
+kubectl -n gitea-runners logs statefulset/gitea-runner -c runner --tail=200 | grep -Eq 'token|GITEA_RUNNER_REGISTRATION_TOKEN' && exit 1 || true
+```
+
+The `rollout restart` command above is the controlled restart path for this
+StatefulSet. Observe the rollout and each ordinal until all replacement pods are
+Ready; do not delete runner pods directly as part of normal token rotation:
+
+```sh
+kubectl -n gitea-runners wait --for=condition=Ready pod/gitea-runner-0 --timeout=5m
+kubectl -n gitea-runners wait --for=condition=Ready pod/gitea-runner-1 --timeout=5m
+kubectl -n gitea-runners wait --for=condition=Ready pod/gitea-runner-2 --timeout=5m
+kubectl -n gitea-runners wait --for=condition=Ready pod/gitea-runner-3 --timeout=5m
+kubectl -n gitea-runners wait --for=condition=Ready pod/gitea-runner-4 --timeout=5m
+kubectl -n gitea-runners get pods -l app.kubernetes.io/name=gitea-runner -o wide
+```
+
+Verification must confirm the token file mount remains present while the token
+value never appears in logs or evidence:
+
+```sh
+kubectl -n gitea-runners describe pod gitea-runner-0 | grep -n '/runner-secrets\|gitea-runner-token'
+kubectl -n gitea-runners logs pod/gitea-runner-0 -c runner --tail=200 | grep -Eq 'token|GITEA_RUNNER_REGISTRATION_TOKEN' && exit 1 || true
+```
+
+## Deploy and status
+
+These commands are executable only when the external inputs are available:
+
+- `TF_VAR_hcloud_token`
+- `TF_VAR_ssh_public_key`
+- `TF_VAR_ssh_private_key`
+- S3 backend credentials and endpoint access
+- a matching SOPS age identity for `sus/gitea-runners.yaml`
+- `kubectl` access to the target cluster
+- a valid `GCR_NIX_IMAGE_ID` for controller-managed Nix jobs
+
+If any input is missing, stop before `tofu apply`. Do not guess values or reuse
+stale kubeconfig files.
+
+Before production Kubernetes apply or rollout, satisfy both manifest gates:
+
+1. Create or update the `gitea-runner-token` Secret from SOPS. The active
+   Kustomize overlay intentionally does not include a placeholder Secret, but
+   the StatefulSet still mounts `secretName: gitea-runner-token` as
+   `/runner-secrets/token` for `GITEA_RUNNER_REGISTRATION_TOKEN_FILE`.
+2. Keep the persistent-pool ConfigMap labels empty. Runner labels belong to the
+   controller; Nix image readiness is governed by `GCR_NIX_IMAGE_ID`.
+
+Use the same SOPS materialization pattern as token rotation before applying the
+Kubernetes overlay. Applying the namespace alone is allowed so the Secret has a
+target namespace; the full overlay remains gated on the Secret.
+
+```sh
+kubectl apply -f infra/gitea-runners/k8s/namespace.yaml
+umask 077
+token_file=$(mktemp /tmp/gitea-runner-token.XXXXXX)
+trap 'rm -f "$token_file"' EXIT
+sops -d --extract '["gitea"]["hectic-lab"]["org-runner-registration-token"]' sus/gitea-runners.yaml > "$token_file"
+kubectl -n gitea-runners create secret generic gitea-runner-token \
+  --from-file=token="$token_file" \
+  --dry-run=client \
+  -o yaml | kubectl -n gitea-runners apply -f -
+```
+
+Do not run `kubectl apply -k infra/gitea-runners/k8s` until the Secret command
+above succeeds. The persistent pool ConfigMap must retain empty labels.
+
+```sh
+tofu -chdir=infra/gitea-runners/opentofu init
+tofu -chdir=infra/gitea-runners/opentofu validate
+tofu -chdir=infra/gitea-runners/opentofu plan -out=.sisyphus/evidence/task-12-deploy.plan
+tofu -chdir=infra/gitea-runners/opentofu apply .sisyphus/evidence/task-12-deploy.plan
+export KUBECONFIG="$(tofu -chdir=infra/gitea-runners/opentofu output -raw kubeconfig_path)"
+kubectl config current-context
+kubectl get nodes -o wide
+kubectl get sc
+kubectl apply -k infra/gitea-runners/k8s
+kubectl -n gitea-runners get statefulset gitea-runner
+kubectl -n gitea-runners rollout status statefulset/gitea-runner --timeout=10m
+kubectl -n gitea-runners get pods -l app.kubernetes.io/name=gitea-runner -o wide
+kubectl -n gitea-runners get pvc -l app.kubernetes.io/name=gitea-runner -o wide
+kubectl -n gitea-runners get events --sort-by=.lastTimestamp | tail -n 50
+kubectl -n gitea-runners logs statefulset/gitea-runner -c runner --tail=200
+```
+
+Expected status after deploy:
+
+- `kubectl config current-context` names the runner cluster context.
+- `kubectl get nodes -o wide` shows all expected Hetzner nodes Ready.
+- `kubectl get sc` shows the Hetzner CSI storage class used by runner PVCs.
+- `kubectl -n gitea-runners get statefulset gitea-runner` shows 0 desired and 0 ready replicas.
+- `kubectl -n gitea-runners get pvc` shows no active runner PVCs; retained PVCs are rollback-only.
+- The controller host reports healthy and owns runner registrations; no persistent runner claims `ubuntu-latest` or `nix`.
+
+## Legacy rollback pool scaling (manual only)
+
+Persistent-pool scaling is not part of normal operation. Use only after restoring
+its labels and disabling the zero-idle controller as described in `Rollback`.
+
+```sh
+kubectl -n gitea-runners scale statefulset/gitea-runner --replicas=10
+kubectl -n gitea-runners rollout status statefulset/gitea-runner --timeout=10m
+kubectl -n gitea-runners get pods -l app.kubernetes.io/name=gitea-runner -o wide
+kubectl -n gitea-runners get pvc -l app.kubernetes.io/name=gitea-runner -o wide
+
+# Run only workflows supported by restored persistent labels.
+
+kubectl -n gitea-runners scale statefulset/gitea-runner --replicas=5
+kubectl -n gitea-runners rollout status statefulset/gitea-runner --timeout=10m
+kubectl -n gitea-runners get pods -l app.kubernetes.io/name=gitea-runner -o wide
+kubectl -n gitea-runners get pvc -l app.kubernetes.io/name=gitea-runner -o wide
+```
+
+After scaling back down, inspect the cleanup dry-run and deregister any stale
+runner registrations only for pods or PVCs that were intentionally removed.
+
+## Cleanup and stale runner deregistration
+
+Use the dry-run cleanup job to list the StatefulSet, active pods, PVCs, and any
+PVC candidates whose pod is gone. It must not delete active resources.
+
+```sh
+kubectl -n gitea-runners create job gitea-runner-cleanup-dry-run-manual --from=cronjob/gitea-runner-cleanup-dry-run
+kubectl -n gitea-runners wait --for=condition=complete job/gitea-runner-cleanup-dry-run-manual --timeout=2m
+kubectl -n gitea-runners logs job/gitea-runner-cleanup-dry-run-manual -c cleanup-dry-run
+```
+
+Pool-wide DinD storage checks and cleanup:
+
+```sh
+for pod in $(kubectl -n gitea-runners get pods -l app.kubernetes.io/name=gitea-runner -o jsonpath='{range .items[*]}{.metadata.name}{"\n"}{end}'); do
+  kubectl -n gitea-runners exec "pod/${pod}" -c docker -- docker system df
+done
+
+for pod in $(kubectl -n gitea-runners get pods -l app.kubernetes.io/name=gitea-runner -o jsonpath='{range .items[*]}{.metadata.name}{"\n"}{end}'); do
+  kubectl -n gitea-runners exec "pod/${pod}" -c docker -- docker system prune --all --force --filter until=24h
+done
+
+for pod in $(kubectl -n gitea-runners get pods -l app.kubernetes.io/name=gitea-runner -o jsonpath='{range .items[*]}{.metadata.name}{"\n"}{end}'); do
+  kubectl -n gitea-runners exec "pod/${pod}" -c docker -- docker system df
+done
+```
+
+If a PVC such as `data-gitea-runner-3` is intentionally deleted, the matching
+pod loses `/data/.runner`. Deregister that runner from Gitea, or let the
+replacement pod re-register intentionally with the current organization token.
+Never delete an active runner PVC as routine cleanup.
+
+Non-UI Gitea registration reconciliation uses an admin token stored outside this
+repository:
+
+```sh
+kubectl -n gitea-runners create secret generic gitea-runner-admin-token --from-file=token=/secure/path/gitea-admin-token
+kubectl -n gitea-runners run gitea-runner-registration-dry-run \
+  --restart=Never \
+  --image=curlimages/curl:8.10.1 \
+  --overrides='{"spec":{"containers":[{"name":"gitea-runner-registration-dry-run","image":"curlimages/curl:8.10.1","command":["/bin/sh","-ec","umask 077; cfg=$(mktemp); trap '\''rm -f \"$cfg\"'\'' EXIT; { printf '\''header = \"Authorization: token '\''; cat /admin-token/token; printf '\''\"\\n'\''; printf '\''url = \"https://gitea.hectic-lab.com/api/v1/orgs/hectic-lab/actions/runners\"\\n'\''; } > \"$cfg\"; curl -fsS --config \"$cfg\""],"volumeMounts":[{"name":"admin-token","mountPath":"/admin-token","readOnly":true}]}],"volumes":[{"name":"admin-token","secret":{"secretName":"gitea-runner-admin-token","defaultMode":256}}]}}'
+kubectl -n gitea-runners logs pod/gitea-runner-registration-dry-run
+```
+
+Delete the temporary `gitea-runner-admin-token` Secret only after the dry-run
+pod has completed and its logs have been collected.
+
+After the dry-run list identifies a stale registration and the PVC or pod
+deletion has been recorded, remove that exact Gitea runner by id through the
+API:
+
+```sh
+runner_id='REPLACE_WITH_STALE_RUNNER_ID'
+umask 077
+curl_config=$(mktemp /tmp/gitea-runner-admin-curl.XXXXXX)
+trap 'rm -f "$curl_config"' EXIT
+{
+  printf 'request = "DELETE"\n'
+  printf 'header = "Authorization: token '
+  cat /secure/path/gitea-admin-token
+  printf '"\n'
+  printf 'url = "https://gitea.hectic-lab.com/api/v1/orgs/hectic-lab/actions/runners/%s"\n' "$runner_id"
+} > "$curl_config"
+curl -fsS --config "$curl_config"
+```
+
+Do not run the delete command for a runner that still has an active
+`gitea-runner-*` pod or retained `data-gitea-runner-*` PVC unless that PVC is
+being intentionally reset for re-registration.
+
+## Legacy Kubernetes application rollback
+
+Rollback the app layer only. Do not use this section to destroy the cluster.
+
+```sh
+kubectl -n gitea-runners rollout history statefulset/gitea-runner
+kubectl -n gitea-runners rollout undo statefulset/gitea-runner --to-revision=<known-good-revision>
+kubectl -n gitea-runners rollout status statefulset/gitea-runner --timeout=10m
+kubectl -n gitea-runners get pods -l app.kubernetes.io/name=gitea-runner -o wide
+kubectl -n gitea-runners get pvc -l app.kubernetes.io/name=gitea-runner -o wide
+kubectl -n gitea-runners logs statefulset/gitea-runner -c runner --tail=200
+```
+
+If a manifest rollback is needed, reapply the repo overlay after checking out the
+known-good revision, then re-run the rollout checks:
+
+```sh
+kubectl -n gitea-runners apply -k infra/gitea-runners/k8s
+kubectl -n gitea-runners rollout status statefulset/gitea-runner --timeout=10m
+```
+
+## Full cluster teardown
+
+This destroys Hetzner resources owned by the kube-hetzner stack, including the
+`gitea-runners` cluster nodes, the `control-plane` node pool, the
+`runner-workers` node pool, the cluster network, load balancer resources,
+firewall objects, and any attached Hetzner CSI volumes still managed by the
+stack. Do not run teardown unless the destruction is intentional.
+
+```sh
+tofu -chdir=infra/gitea-runners/opentofu plan -destroy -out=.sisyphus/evidence/task-12-destroy.plan
+tofu -chdir=infra/gitea-runners/opentofu show -no-color .sisyphus/evidence/task-12-destroy.plan
+tofu -chdir=infra/gitea-runners/opentofu apply .sisyphus/evidence/task-12-destroy.plan
+```
+
+## Partial OpenTofu apply recovery
+
+If `tofu apply` fails after creating some resources, do not destroy blindly.
+First reconcile state and inspect what the stack thinks exists:
+
+```sh
+tofu -chdir=infra/gitea-runners/opentofu plan -refresh-only -out=.sisyphus/evidence/task-12-refresh.plan
+tofu -chdir=infra/gitea-runners/opentofu show -no-color .sisyphus/evidence/task-12-refresh.plan
+tofu -chdir=infra/gitea-runners/opentofu state list
+```
+
+Then rerun the normal plan path. Use `-target` only as a last resort when a
+single resource is stuck and the drift is understood.
+
+## S3 backend recovery
+
+If backend init or state access fails, first verify the bucket and versioning
+outside OpenTofu, then reconfigure the backend:
+
+```sh
+nix run nixpkgs#awscli2 -- s3api head-bucket --bucket gitea-runner-hectic-lab
+nix run nixpkgs#awscli2 -- s3api get-bucket-versioning --bucket gitea-runner-hectic-lab
+tofu -chdir=infra/gitea-runners/opentofu init -reconfigure
+tofu -chdir=infra/gitea-runners/opentofu plan
+```
+
+If the backend reports a stale lock, confirm no `tofu` process is active, then
+use `tofu force-unlock <LOCK_ID>` with the lock id from the error. Never force
+unlock a live plan or apply.
+
+## Gitea outage troubleshooting
+
+Use the public HTTPS service, not `localhost` or `127.0.0.1` inside job
+containers.
+
+```sh
+kubectl -n gitea-runners run gitea-outage-probe --rm --restart=Never --image=curlimages/curl:8.10.1 -- curl -fsS https://gitea.hectic-lab.com/api/healthz
+kubectl -n gitea-runners logs statefulset/gitea-runner -c runner --tail=200 | grep -E 'connection refused|timeout|tls|certificate|temporary failure' || true
+kubectl -n gitea-runners get events --sort-by=.lastTimestamp | tail -n 50
+```
+
+If Gitea is down, keep the existing StatefulSet and PVCs intact. Do not delete
+`/data/.runner` just because the service is unavailable. Once Gitea returns,
+repeat the token rotation or re-registration path if a pod restarted while the
+service was unavailable and lost its runner identity.
+
+## Release checklist
+
+Do not release unless the following evidence files exist and are readable:
+
+- `.sisyphus/evidence/task-5-cluster-plan.txt`
+- `.sisyphus/evidence/task-5-secret-plan-scan.txt`
+- `.sisyphus/evidence/task-9-deploy.txt`
+- `.sisyphus/evidence/task-9-secret-mount.txt`
+- `.sisyphus/evidence/task-10-ubuntu-workflow.txt`
+- `.sisyphus/evidence/task-10-nix-workflow.txt`
+- `.sisyphus/evidence/task-11-scale.txt`
+- `.sisyphus/evidence/task-11-restart-cleanup.txt`
+
+If any evidence file is missing, stop and collect it before treating the runbook
+as complete.
+
+## Ephemeral VM runner cutover
+
+This section governs replacing the legacy persistent K8s runner pool with the
+ephemeral-VM controller (`package/gitea-runner-controller`) on this host.
+The K8s pool above remains rollback-only until cutover is explicitly accepted.
+
+### Operator gates (all three required before enable)
+
+1. Secrets — add to `sus/gitea-runners.yaml` under
+   `gitea/hectic-lab/controller/*`:
+
+   ```sh
+   sops sus/gitea-runners.yaml
+   # add keys:
+   #   gitea:
+   #     hectic-lab:
+   #       controller:
+   #         hcloud-token:    <Hetzner API token, VM create/destroy scope>
+   #         webhook-secret:  <random 32+ bytes; also set as Gitea webhook secret>
+   #         admin-token:     <Gitea token with admin:runner scope for stale cleanup>
+   ```
+
+   The org registration token key
+   `gitea/hectic-lab/org-runner-registration-token` already exists and is reused.
+
+2. Base image — build the MicroOS snapshot and record its id:
+
+   ```sh
+   nix develop .#gitea-runners -c gitea-runners-build-microos-snapshots x86
+   hcloud image list --selector '' -o json | jq '.[] | select(.type=="snapshot")'
+   ```
+
+3. DNS — A record `runners.hectic-lab.com -> 128.140.75.58` (ACME needs it).
+
+### Enable
+
+```sh
+# nixos/system/hectic-lab/hectic-lab.nix: resolve the FIXME block
+hectic.services.gitea-runner-controller = {
+  enable  = true;
+  imageId = "<snapshot-id-from-gate-2>";
+};
+nixos-rebuild --target root@128.140.75.58 switch
+systemctl status gitea-runner-controller.service gitea-runner-webhook.service
+```
+
+### Register the Gitea webhook
+
+Org-level (preferred) or per-repo, on `https://gitea.hectic-lab.com`:
+
+- URL: `https://runners.hectic-lab.com/`
+- Method: `POST`, content type: JSON
+- Secret: value of `gitea/hectic-lab/controller/webhook-secret`
+- Trigger events: `Workflow jobs` only (`workflow_job`)
+
+### Long CUDA/Magma deployment time budgets
+
+The `deploy-neuro` workflow uses these nested limits:
+
+| Layer | Limit |
+| --- | --- |
+| Wrapped build/deploy command | 6 hours (`WITH_ATTIC_BUILD_TIMEOUT=21600`) |
+| Final cache drain | 1 hour (`WITH_ATTIC_DRAIN_TIMEOUT=3600`) |
+| Workflow job | 435 minutes, including 15 minutes of setup/cleanup margin |
+| `gross-nix-x86-perf` runner | 480 minutes |
+| `gross-nix-x86-highmem` runner | 480 minutes |
+| Gitea `actions.ENDLESS_TASK_TIMEOUT` | 8 hours |
+| VM hard lifetime from allocation | 480 minutes; no destruction grace |
+
+`ubuntu-latest` keeps a 180-minute limit; `nix` uses a 480-minute limit for
+long-running Nix deployments. Deploy the controller and Gitea watchdog settings
+before dispatching the longer workflow. Already
+allocated VMs retain the TTL and runner configuration assigned when they were
+created; updating the controller does not extend a running job.
+
+`gross-nix-x86-highmem` is an explicit costly high-memory escape hatch backed
+only by Hetzner CCX53 in `nbg1`, `fsn1`, or `hel1`; it may fall back by region
+only, never to a lower-RAM server type. Current Hetzner public pricing for
+Germany/Finland CCX53 is 0.8550 EUR/hour excluding IPv4, so one 480-minute
+allocation reserves 6.84 EUR against the controller budget before VM creation.
+
+These are maximum lifetimes. Failed, cancelled, skipped, and unbootstrapped jobs
+still trigger immediate VM teardown; failed jobs retain pre-destroy diagnostics.
+After a successful job, its bootstrapped VM stays running until next hourly
+boundary measured from original VM creation, capped by profile TTL. Same-repo,
+same-label queued work can atomically claim that idle VM. Reuse preserves
+original Hetzner labels and runner name, creates no server, fetches no new
+registration token, and makes no second budget reservation. Idle VMs are still
+billed: controller deletes them at slot expiry and never relies on stopping a
+server to avoid charges.
+
+Reuse retains runner host filesystem and registration identity. It is therefore
+restricted to same repository and exact label inside this trusted-only pool;
+allowed repositories must not run untrusted fork or pull-request code. A failed
+or otherwise non-successful job is never reused.
+
+Active and idle VMs are deleted at profile hard TTL without grace. Idle reuse is
+allowed only when at least one configured reconcile interval remains before both
+slot expiry and hard TTL. Controller budget reservation still uses full label TTL
+on initial creation, so a long-running label reserves more of existing monthly
+budget. Do not raise that budget or disable timeout safeguards just to bypass a
+refused allocation.
+
+After changing any timeout, verify the complete chain rather than only
+`timeout-minutes`; a shorter wrapper, runner, server watchdog, or VM TTL wins.
+
+### Pre-flight verification (before first real job)
+
+```sh
+curl -fsS https://runners.hectic-lab.com/ -o /dev/null -w '%{http_code}\n' # any 4xx from handler = reachable
+journalctl -u gitea-runner-webhook -n 20 --no-pager
+hcloud server list -o json | jq '[.[] | select(.labels["gitea-runner-controller"]=="managed")] | length' # expect 0
+```
+
+Zero managed VMs is expected after retained billing slots expire. Immediately
+after successful work, one managed VM per retained profile may remain until its
+recorded hourly boundary.
+
+### End-to-end acceptance (Task 9)
+
+Trigger `.gitea/workflows/runner-nix-smoke.yaml` via workflow_dispatch, then:
+
+```sh
+watch_labels() { hcloud server list -o json | jq '[.[] | select(.labels["gitea-runner-controller"]=="managed") | {id,name,labels}]'; }
+watch_labels                                   # exactly one VM while queued/running
+journalctl -f -u gitea-runner-controller       # vm-created / vm-destroyed events
+watch_labels                                   # VM may remain until next hourly boundary
+curl -fsS -H "Authorization: token $ADMIN" \
+  https://gitea.hectic-lab.com/api/v1/orgs/hectic-lab/actions/runners \
+  | jq '[.entries[] | select(.name | startswith("gcr-"))] | length'  # may remain during retained slot
+```
+
+Failure paths to verify identically: duplicate delivery (send same webhook twice
+via Gitea UI "Test delivery"), cancelled job, unknown-label job.
+
+### Rollback
+
+K8s rollback pool now defaults to deleted state:
+
+- `infra/gitea-runners/k8s/statefulset.yaml` keeps `replicas: 0`
+- old kube-hetzner nodes may be deleted to preserve zero idle cost
+- PVCs and IaC remain for manual rollback only
+
+Re-enable sequence:
+
+```sh
+# 1. edit nixos/system/hectic-lab/hectic-lab.nix and set
+#    services.gitea-runner-controller.enable = false, then rebuild:
+nixos-rebuild --target root@128.140.75.58 switch
+
+# 2. reprovision old kube-hetzner nodes when they were deleted:
+tofu -chdir=infra/gitea-runners/opentofu apply
+
+# 3. while controller is disabled, destroy every surviving managed VM and
+#    verify no gcr-* runner registration remains online:
+hcloud server list -o json \
+  | jq -r '.[] | select(.labels["gitea-runner-controller"]=="managed") | .id' \
+  | xargs -r -n1 hcloud server delete
+
+# 4. restore kubeconfig / cluster access, restore `ubuntu-latest` in the
+#    ConfigMap labels, then re-enable K8s runner pool. The legacy image does not
+#    provide `nix`; do not dispatch Nix workflows until a Nix-capable K8s image
+#    and label mapping are restored:
+kubectl -n gitea-runners edit configmap/gitea-runner-config
+kubectl -n gitea-runners scale statefulset/gitea-runner --replicas=5
+kubectl -n gitea-runners rollout status statefulset/gitea-runner --timeout=10m
+```
+
+Any managed VM or `gcr-*` registration found after step 3 must be removed before
+restoring persistent labels; otherwise both pools can claim the same job.
@@ -26,12 +26,39 @@
    "aarch64-darwin"
  ];

+  cudaUnfreeNames = [
+    "cuda_nvcc"
+    "cuda_cudart"
+    "cuda_cuobjdump"
+    "cuda_cupti"
+    "cuda_nvdisasm"
+    "cuda_cccl"
+    "cuda_nvml_dev"
+    "cuda_nvrtc"
+    "cuda_nvtx"
+    "cuda_profiler_api"
+
+    "libcusparse_lt"
+    "libcublas"
+    "libcufft"
+    "libcufile"
+    "libcurand"
+    "libcusolver"
+    "libnvjitlink"
+    "libcusparse"
+    "cudnn"
+  ];
+
+  cudaUnfreePredicate = pkg:
+    builtins.elem (nixpkgs.lib.getName pkg) cudaUnfreeNames;
+
  forSystemsWithPkgs = supportedSystems: pkgOverlays: f:
    builtins.foldl' (
      acc: system: let
        pkgs = import nixpkgs {
          inherit system;
          overlays = pkgOverlays;
+          config.allowUnfreePredicate = cudaUnfreePredicate;
        };
        systemOutputs = f {
          system = system;
@@ -58,7 +85,7 @@
    else {};
 in {
  # -- For all systems --
-  inherit dotEnv minorEnvironment parseEnv forAllSystemsWithPkgs forSystemsWithPkgs commonSystems AllSystems;
+  inherit dotEnv minorEnvironment parseEnv forAllSystemsWithPkgs forSystemsWithPkgs commonSystems AllSystems cudaUnfreeNames cudaUnfreePredicate;

  forSystems = systems: nixpkgs.lib.genAttrs systems;
  forAllSystems = nixpkgs.lib.genAttrs AllSystems;
@@ -67,6 +94,7 @@ in {
    logs = builtins.readFile ./shell/logs.sh;
    check-tool = builtins.readFile ./shell/check-tool.sh;
    local-dir = builtins.readFile ./shell/local-dir.sh;
+    load-sops = builtins.readFile ./shell/load-sops.sh;
  };

  sharedShellAliases = {
@@ -98,11 +126,11 @@ in {
    else throw (envErrorMessage varName);

  # -- Cargo.toml --
-  cargoToml = src: (builtins.fromTOML (builtins.readFile "${src}/Cargo.toml"));
+  cargoToml = manifest: (builtins.fromTOML (builtins.readFile manifest));

  # Consolidated SQL bundles for the `hectic` schema. Single source of truth
  # for everything that creates objects in the `hectic` namespace, used by
-  # migrator (init-time) and db-tool (postgres-init + hydrate). Consumers apply
+  # migrator (init-time), db-dev/database hydrate, and db-ops secrets loading. Consumers apply
  # the full bundle via lib/hook/apply-hectic-bundle.sh.
  #
  # The whole hectic system shares one `versionString`; `hectic-version.sql`
@@ -115,19 +143,42 @@ in {
  hectic = let
    versionString = lib.fileContents ./hook/sql/HECTIC_VERSION;
    static = path: { inherit path; sql = builtins.readFile path; };
-    templated = path: {
+    templated = path: let
      sql = builtins.replaceStrings
        [ "@HECTIC_VERSION@" ]
        [ versionString ]
        (builtins.readFile path);
+    in {
+      inherit sql;
+      path = builtins.toFile (builtins.baseNameOf (toString path)) sql;
    };
-  in {
+  in rec {
    inherit versionString;
    version     = templated ./hook/sql/hectic-version.sql;
    secret      = static    ./hook/sql/hectic-secret.sql;
    migration   = static    ./hook/sql/hectic-migration.sql;
    inheritance = static    ./hook/sql/hectic-inheritance.sql;
-    applyBundleScript = ./hook/apply-hectic-bundle.sh;
+    bundleFiles = [
+      version.path
+      secret.path
+      migration.path
+      inheritance.path
+    ];
+    applyBundleScript =
+      builtins.replaceStrings
+        [
+          "@HECTIC_VERSION_SQL@"
+          "@HECTIC_SECRET_SQL@"
+          "@HECTIC_MIGRATION_SQL@"
+          "@HECTIC_INHERITANCE_SQL@"
+        ]
+        [
+          "${version.path}"
+          "${secret.path}"
+          "${migration.path}"
+          "${inheritance.path}"
+        ]
+        (builtins.readFile ./hook/apply-hectic-bundle.sh);
  };

  # Back-compat alias. Prefer `self.lib.hectic.inheritance`.
@@ -164,19 +215,27 @@ in {
  readModulesRecursive' = path: extraArgs:
    with lib;
    with builtins; let
-      paths = pipe "${path}" [
-        (filesystem.listFilesRecursive)
-        (filter (hasSuffix ".nix"))
-      ];
+      collectPaths = dir: prefix:
+        concatLists (mapAttrsToList (name: type: let
+          path' = dir + "/${name}";
+          name' = if prefix == "" then name else "${prefix}/${name}";
+        in
+          if type == "directory"
+          then collectPaths path' name'
+          else [{
+            inherit path';
+            name = name';
+          }]
+        ) (readDir dir));
+      paths = filter (path': hasSuffix ".nix" path'.name) (collectPaths path "");
      pathToName = flip pipe [
-        (removePrefix "${path}/")
        (replaceStrings ["/" ".nix"] ["." ""])
        (removeSuffix ".nix")
      ];
      attrList =
        map (path': {
-          name = pathToName (unsafeDiscardStringContext path');
-          value = import path' extraArgs;
+          name = pathToName path'.name;
+          value = import path'.path' extraArgs;
        })
        paths;
    in
@@ -7,17 +7,12 @@
 #
 # Idempotent: each SQL file uses IF NOT EXISTS / CREATE OR REPLACE.
 #
-# Required env (caller injects from Nix):
-#   HECTIC_VERSION_SQL      - path to hectic-version.sql (substituted)
-#   HECTIC_SECRET_SQL       - path to hectic-secret.sql
-#   HECTIC_MIGRATION_SQL    - path to hectic-migration.sql
-#   HECTIC_INHERITANCE_SQL  - path to hectic-inheritance.sql
-#
 # Usage:
 #   apply_hectic_bundle <PGURL> [<DOTENV_CONTENT>]
 #
-# If DOTENV_CONTENT is non-empty, it is loaded into hectic.secret via
-# hectic.load_secrets_from_env() after the bundle is applied.
+# If DOTENV_CONTENT is non-empty, it is base64-encoded and then loaded into
+# hectic.secret via hectic.load_secrets_from_env() after the bundle is applied.
+# SQL file paths are substituted by Nix evaluation time.

 apply_hectic_bundle() {
  pgurl="${1:-}"
@@ -28,29 +23,27 @@ apply_hectic_bundle() {
    return 3
  fi

-  for var in HECTIC_VERSION_SQL HECTIC_SECRET_SQL HECTIC_MIGRATION_SQL HECTIC_INHERITANCE_SQL; do
-    eval "val=\${$var:-}"
-    if [ -z "$val" ]; then
-      printf '%s\n' "apply-hectic-bundle: $var not set" >&2
-      return 3
-    fi
-    if [ ! -r "$val" ]; then
-      printf '%s\n' "apply-hectic-bundle: $var not readable: $val" >&2
+  set -- \
+    "@HECTIC_VERSION_SQL@" \
+    "@HECTIC_SECRET_SQL@" \
+    "@HECTIC_MIGRATION_SQL@" \
+    "@HECTIC_INHERITANCE_SQL@"
+
+  for sql_path do
+    if [ ! -r "$sql_path" ]; then
+      printf '%s\n' "apply-hectic-bundle: SQL file not readable: $sql_path" >&2
      return 1
    fi
  done

-  psql "$pgurl" -v ON_ERROR_STOP=1 -f "$HECTIC_VERSION_SQL"     || return 1
-  psql "$pgurl" -v ON_ERROR_STOP=1 -f "$HECTIC_SECRET_SQL"      || return 1
-  psql "$pgurl" -v ON_ERROR_STOP=1 -f "$HECTIC_MIGRATION_SQL"   || return 1
-  psql "$pgurl" -v ON_ERROR_STOP=1 -f "$HECTIC_INHERITANCE_SQL" || return 1
+  for sql_path do
+    psql "$pgurl" -v ON_ERROR_STOP=1 -f "$sql_path" || return 1
+  done

  if [ -n "$env_content" ]; then
-    # Dollar-quote with $ps_env$ tag to preserve all content verbatim.
+    env_content_b64="$(printf '%s' "$env_content" | base64 | tr -d '\n')" || return 1
    psql "$pgurl" -v ON_ERROR_STOP=1 <<SQL || return 1
-SELECT hectic.load_secrets_from_env(\$ps_env\$
-$env_content
-\$ps_env\$);
+SELECT hectic.load_secrets_from_env(convert_from(decode('$env_content_b64', 'base64'), 'UTF8'));
 SQL
  fi

@@ -4,8 +4,8 @@ Single source of truth for every object created in the `hectic` PostgreSQL
 schema. Consumed by:

 - `package/migrator` — applies the bundle on `migrator init` (mandatory).
- `package/db-tool` — applies the bundle in `database hydrate` (default; opt
-  out with `--no-hook`).
+- `package/db-tool` — applies the bundle in `db-dev` / `database hydrate`
+  (default; opt out with `--no-hook`) and in `db-ops secrets load`.
 - External consumers (e.g. `proxydoe`) — invoke `psql -f` directly against the
  paths exposed via `self.lib.hectic.*.path`.

@@ -42,23 +42,26 @@ that already matches.
 ```nix
 self.lib.hectic = {
  versionString;          # e.g. "0.1.0"
-  version     = { sql; };           # templated
+  version     = { sql; path; };     # templated
  secret      = { sql; path; };
  migration   = { sql; path; };
  inheritance = { sql; path; };
-  applyBundleScript;                # ./hook/apply-hectic-bundle.sh
+  bundleFiles;                      # ordered bundle file paths
+  applyBundleScript;                # generated helper shell source with paths embedded
 };
 ```

 `.sql` is the file contents as a string. `.path` is the Nix store path of the
-verbatim source (only available on non-templated entries; consumers needing a
-materialized version of `version.sql` must do
-`pkgs.runCommand "hectic-version.sql" { text = self.lib.hectic.version.sql; passAsFile = ["text"]; } ''cp "$textPath" "$out"''`).
+materialized file to pass to `psql -f`. `version.path` is generated at Nix
+evaluation time from the templated SQL; the other `*.path` entries point at the
+verbatim source files in the store.

 ## Shell helper (`apply-hectic-bundle.sh`)

-`lib/hook/apply-hectic-bundle.sh` is a dash-compatible helper sourced by both
-`migrator` and `db-tool`. Public entry point:
+`lib/hook/apply-hectic-bundle.sh` is a dash-compatible helper template.
+`self.lib.hectic.applyBundleScript` is the generated shell source with concrete
+SQL paths embedded at Nix evaluation time. `migrator`, `db-dev`, and `db-ops` splice that
+shell source directly into their generated scripts. Public entry point:

 ```sh
 apply_hectic_bundle <PGURL> [<DOTENV_CONTENT>]
@@ -70,28 +73,23 @@ apply_hectic_bundle <PGURL> [<DOTENV_CONTENT>]
  dollar-quoted (`$ps_env$`) string so secret values cannot terminate the
  literal.

-Required environment (paths to the SQL files):
-
- `HECTIC_VERSION_SQL`
- `HECTIC_SECRET_SQL`
- `HECTIC_MIGRATION_SQL`
- `HECTIC_INHERITANCE_SQL`
-
-`migrator` and `db-tool` set these via Nix at build time. External consumers
-typically invoke `psql -f` against the paths directly instead of sourcing the
-helper.
+The SQL file paths are embedded into the helper at Nix evaluation time, so
+callers only need to source the generated script and call the function.
+External consumers that do not want to source the helper can still invoke
+`psql -f` against `self.lib.hectic.bundleFiles` or the individual
+`self.lib.hectic.*.path` entries directly.

 ## Adding a new SQL file

 1. Add `lib/hook/sql/hectic-<name>.sql`.
 2. Wire it into `lib/default.nix` under `lib.hectic.<name>`.
-3. Inject `HECTIC_<NAME>_SQL` in both `package/migrator/default.nix` and
-   `package/db-tool/default.nix`.
-4. Append a `psql -f "$HECTIC_<NAME>_SQL"` step to
-   `lib/hook/apply-hectic-bundle.sh` in the correct order.
+3. Add its `.path` to `lib.hectic.bundleFiles` in the correct order.
+4. Add a matching placeholder/replacement in `lib.hectic.applyBundleScript` and
+   update `lib/hook/apply-hectic-bundle.sh` to apply the file.
 5. Bump `HECTIC_VERSION` if the new content changes existing semantics.
-6. Update tests in `test/package/migrator/test/postgresql/init-hectic-bundle/`
-   and `test/package/db-tool/test/postgresql/hydrate-hook/`.
+6. Update tests in `test/package/migrator/test/postgresql/init-hectic-bundle/`,
+   `test/package/db-tool/test/postgresql/hydrate-hook/`, and any `db-ops`
+   bundle-loading coverage.

 ## Versioning

@@ -0,0 +1,204 @@
+load_sops_shell_quote() {
+  printf "'"
+  printf '%s' "$1" | sed "s/'/'\"'\"'/g"
+  printf "'"
+}
+
+load_sops_normalize_key() {
+  load_sops_normalized_key=$(printf '%s' "$1" | tr '.-' '__' | tr '[:lower:]' '[:upper:]')
+
+  case "$load_sops_normalized_key" in
+    ''|[!A-Z_]*|*[!A-Z0-9_]*)
+      printf 'load-sops: invalid environment name after key normalization\n' >&2
+      return 1
+      ;;
+  esac
+
+  printf '%s\n' "$load_sops_normalized_key"
+}
+
+load_sops_env_from_sops_file() {
+  load_sops_file=$1
+  load_sops_extract=${2-}
+
+  if ! command -v sops >/dev/null 2>&1; then
+    printf 'load-sops: required tool `sops` not found\n' >&2
+    return 1
+  fi
+
+  if ! command -v yq >/dev/null 2>&1; then
+    printf 'load-sops: required tool `yq` not found\n' >&2
+    return 1
+  fi
+
+  load_sops_decrypted=''
+  load_sops_attempt=0
+  load_sops_max_retries=${LOAD_SOPS_MAX_RETRIES:-1}
+  load_sops_prompt=${LOAD_SOPS_PROMPT:-0}
+
+  while :; do
+    if [ -n "$load_sops_extract" ]; then
+      if load_sops_decrypted=$(sops -d --extract "$load_sops_extract" "$load_sops_file" 2>/dev/null); then
+        load_sops_status=0
+      else
+        load_sops_status=$?
+      fi
+    else
+      if load_sops_decrypted=$(sops -d "$load_sops_file" 2>/dev/null); then
+        load_sops_status=0
+      else
+        load_sops_status=$?
+      fi
+    fi
+
+    if [ "$load_sops_status" -eq 0 ]; then
+      break
+    fi
+
+    if [ "$load_sops_prompt" != 1 ]; then
+      printf 'load-sops: failed to decrypt file\n' >&2
+      return 1
+    fi
+
+    if ! [ -t 0 ] || ! [ -r /dev/tty ]; then
+      printf 'load-sops: decrypt failed and prompt requested, but no TTY is available\n' >&2
+      return 1
+    fi
+
+    load_sops_attempt=$((load_sops_attempt + 1))
+    if [ "$load_sops_max_retries" != 0 ] && [ "$load_sops_attempt" -gt "$load_sops_max_retries" ]; then
+      printf 'load-sops: decrypt failed after configured retries\n' >&2
+      return 1
+    fi
+
+    load_sops_use_script=${LOAD_SOPS_USE_SCRIPT:-auto}
+    load_sops_quoted_file=$(load_sops_shell_quote "$load_sops_file") || return 1
+    load_sops_quoted_tty=$(load_sops_shell_quote "$(tty)") || return 1
+    case "$load_sops_use_script" in
+      auto)
+        if command -v script >/dev/null 2>&1 && [ -t 0 ]; then
+          if script -qefc "env GPG_TTY=$load_sops_quoted_tty sops --output /dev/null -d $load_sops_quoted_file" /dev/null >/dev/null 2>&1; then
+            load_sops_script_status=0
+          else
+            load_sops_script_status=$?
+          fi
+          if [ "$load_sops_script_status" -eq 0 ]; then
+            continue
+          fi
+        fi
+        ;;
+      1)
+        if ! command -v script >/dev/null 2>&1; then
+          printf 'load-sops: required tool `script` not found\n' >&2
+          return 1
+        fi
+        if script -qefc "env GPG_TTY=$load_sops_quoted_tty sops --output /dev/null -d $load_sops_quoted_file" /dev/null >/dev/null 2>&1; then
+          load_sops_script_status=0
+        else
+          load_sops_script_status=$?
+        fi
+        if [ "$load_sops_script_status" -eq 0 ]; then
+          continue
+        fi
+        ;;
+      0)
+        ;;
+      *)
+        printf 'load-sops: LOAD_SOPS_USE_SCRIPT must be auto, 0, or 1\n' >&2
+        return 1
+        ;;
+    esac
+
+    printf 'load-sops: enter SOPS_AGE_KEY_CMD: ' >/dev/tty
+    if ! IFS= read -r SOPS_AGE_KEY_CMD </dev/tty; then
+      printf 'load-sops: failed to read prompt input\n' >&2
+      return 1
+    fi
+    export SOPS_AGE_KEY_CMD
+  done
+
+  load_sops_env_from_yaml_text "$load_sops_decrypted"
+}
+
+load_sops_env_from_yaml_file() {
+  load_sops_file=$1
+
+  if ! command -v yq >/dev/null 2>&1; then
+    printf 'load-sops: required tool `yq` not found\n' >&2
+    return 1
+  fi
+
+  load_sops_env_from_yaml_text "$(cat "$load_sops_file")"
+}
+
+load_sops_env_from_yaml_text() {
+  load_sops_yaml=$1
+  load_sops_seen=''
+
+  if load_sops_keys=$(printf '%s' "$load_sops_yaml" | yq -r 'keys | .[]' 2>/dev/null); then
+    load_sops_keys_status=0
+  else
+    load_sops_keys_status=$?
+  fi
+
+  if [ "$load_sops_keys_status" -ne 0 ]; then
+    printf 'load-sops: failed to inspect YAML top-level keys\n' >&2
+    return 1
+  fi
+
+  while IFS= read -r load_sops_key; do
+    [ -n "$load_sops_key" ] || continue
+
+    load_sops_name=$(load_sops_normalize_key "$load_sops_key") || return 1
+
+    case "
+$load_sops_seen
+" in
+      *"
+$load_sops_name
+"*)
+        if [ "${LOAD_SOPS_ALLOW_COLLISIONS:-0}" != 1 ]; then
+          printf 'load-sops: normalized environment name collision\n' >&2
+          return 1
+        fi
+        ;;
+    esac
+    load_sops_seen=${load_sops_seen}${load_sops_seen:+"
+"}$load_sops_name
+
+    load_sops_kind=$(printf '%s' "$load_sops_yaml" | yq -r '."'"$load_sops_key"'" | kind' 2>/dev/null) || {
+      printf 'load-sops: failed to inspect YAML value kind\n' >&2
+      return 1
+    }
+    load_sops_tag=$(printf '%s' "$load_sops_yaml" | yq -r '."'"$load_sops_key"'" | tag' 2>/dev/null) || {
+      printf 'load-sops: failed to inspect YAML value tag\n' >&2
+      return 1
+    }
+
+    if [ "$load_sops_kind" != scalar ]; then
+      printf 'load-sops: top-level YAML values must be scalars\n' >&2
+      return 1
+    fi
+
+    if [ "$load_sops_tag" = '!!null' ]; then
+      printf 'load-sops: top-level YAML null values are not supported\n' >&2
+      return 1
+    fi
+
+    if [ "${LOAD_SOPS_OVERWRITE:-1}" = 0 ]; then
+      eval 'load_sops_already_set=${'"$load_sops_name"'+x}'
+      if [ -n "$load_sops_already_set" ]; then
+        continue
+      fi
+    fi
+
+    load_sops_value=$(printf '%s' "$load_sops_yaml" | yq -r '."'"$load_sops_key"'"' 2>/dev/null) || {
+      printf 'load-sops: failed to read YAML scalar value\n' >&2
+      return 1
+    }
+    load_sops_quoted=$(load_sops_shell_quote "$load_sops_value") || return 1
+    eval "export $load_sops_name=$load_sops_quoted"
+  done <<EOF
+$load_sops_keys
+EOF
+}
@@ -9,11 +9,11 @@ with self.lib;
 let
  # Combine hectic modules into one
  hectic.imports = attrValues (
-    readModulesRecursive' ./hectic { inherit flake self inputs; }
+    readModulesRecursive' (flake + "/nixos/module/hectic") { inherit flake self inputs; }
  );
  # Read generic modules separately
  generic = readModulesRecursive'
-    ./generic
+    (flake + "/nixos/module/generic")
    { inherit flake self inputs; };
 in generic // {
  inherit hectic;
@@ -18,6 +18,7 @@
  ];

  adminNames = [ "yukkop" ];
+  matrixClusterSopsFile = flake + "/sus/matrix-cluster.yaml";

  cfg = config.hectic.generic.matrix-cluster;
 in {
@@ -39,7 +40,7 @@ in {
        value = {
          key      = "matrix/users/${name}/password";
          owner    = "matrix-synapse";
-          sopsFile = "${flake}/sus/matrix-cluster.yaml";
+          sopsFile = matrixClusterSopsFile;
        };
      }) userNames
    );
@@ -250,6 +250,7 @@ in {
      # failover flip does not need a separate provisioning step.
      systemd.tmpfiles.rules = [
        "d /var/lib/matrix-synapse 0750 matrix-synapse matrix-synapse -"
+        "Z ${s3Cfg.mediaStorePath} 0700 matrix-synapse matrix-synapse -"
      ];

      systemd.services.matrix-cluster-signing-key = {
@@ -319,6 +320,12 @@ in {
          media_store_path = s3Cfg.mediaStorePath;
          signing_key_path = "/var/lib/matrix-synapse/homeserver.signing.key";

+          # Tolerate bursty Element/iPhone presence syncs without disabling limits.
+          rc_presence.per_user = {
+            per_second = 0.5;
+            burst_count = 5;
+          };
+
          experimental_features = {
            msc3266_enabled = true;
            msc4140_enabled = true;
@@ -0,0 +1,56 @@
+{ ... }:
+{ config, lib, pkgs, ... }:
+let
+  cfg = config.services.minecraft-public-relay;
+in {
+  options.services.minecraft-public-relay = {
+    enable = lib.mkEnableOption "restricted SSH relay for Minecraft";
+    publicPort = lib.mkOption { type = lib.types.port; default = 25568; };
+    tunnelPort = lib.mkOption { type = lib.types.port; default = 25577; };
+    publicKey = lib.mkOption {
+      type = lib.types.str;
+      description = "Public SSH key of the Minecraft tunnel client";
+    };
+  };
+  config = lib.mkIf cfg.enable {
+    networking.firewall.allowedTCPPorts = [ cfg.publicPort ];
+    users.groups.mc-wow-relay = { };
+    users.users.mc-wow-relay = {
+      isSystemUser = true;
+      group = "mc-wow-relay";
+      openssh.authorizedKeys.keys = [
+        "restrict,port-forwarding,permitlisten=\"127.0.0.1:${toString cfg.tunnelPort}\" ${cfg.publicKey}"
+      ];
+    };
+    services.openssh.extraConfig = ''
+      Match User mc-wow-relay
+        ClientAliveInterval 15
+        ClientAliveCountMax 3
+        AllowTcpForwarding remote
+        PermitListen 127.0.0.1:${toString cfg.tunnelPort}
+        AllowAgentForwarding no
+        X11Forwarding no
+        PermitTTY no
+        ForceCommand ${pkgs.coreutils}/bin/false
+      Match all
+    '';
+    systemd.sockets.minecraft-wow-proxy = {
+      description = "WorldOfSosal WoW public Minecraft port";
+      wantedBy = [ "sockets.target" ];
+      listenStreams = [ "0.0.0.0:${toString cfg.publicPort}" ];
+    };
+    systemd.services.minecraft-wow-proxy = {
+      description = "Forward Minecraft to the neuro reverse tunnel";
+      requires = [ "minecraft-wow-proxy.socket" ];
+      after = [ "minecraft-wow-proxy.socket" ];
+      serviceConfig = {
+        ExecStart = "${pkgs.systemd}/lib/systemd/systemd-socket-proxyd 127.0.0.1:${toString cfg.tunnelPort}";
+        DynamicUser = true;
+        NoNewPrivileges = true;
+        ProtectSystem = "strict";
+        ProtectHome = true;
+        PrivateTmp = true;
+      };
+    };
+  };
+}
@@ -1,7 +1,7 @@
 { 
  inputs,
-  flake,
  self,
+  ...
 }: {
  pkgs,
  lib,
@@ -27,7 +27,15 @@ in {
    users.defaultUserShell = pkgs.zsh;

    # Enable flakes and new 'nix' command
-    nix.settings.experimental-features = "nix-command flakes";
+    nix.settings = {
+      experimental-features = "nix-command flakes";
+      extra-substituters = [
+        "https://cache.hectic-lab.com/hectic"
+      ];
+      extra-trusted-public-keys = [
+        "hectic:KMQsKow4SoA9K2vOJlOljmx7/Zpf91Yy+5qEtxDDCzA="
+      ];
+    };

    networking.firewall.enable = true;

@@ -1,7 +1,5 @@
-{ 
-  inputs,
-  flake,
-  self,
+{
+  ...
 }:
 { 
  pkgs,
@@ -11,6 +9,13 @@
 }: let
  cfg = config.hectic.hardware.hetzner-cloud;
  isNewer = cfg.generation == "newer";
+  networkMatchConfig =
+    (lib.optionalAttrs (cfg.networkMatchConfigName != null) {
+      Name = cfg.networkMatchConfigName;
+    })
+    // (lib.optionalAttrs (cfg.networkMatchConfigMac != null) {
+      PermanentMACAddress = cfg.networkMatchConfigMac;
+    });
 in {
  options.hectic.hardware.hetzner-cloud = {
    enable = lib.mkEnableOption "Enable hetzner-cloud hardware configurations";
@@ -51,6 +56,14 @@ in {
        
      '';
    };
+    floatingIpv4 = lib.mkOption {
+      type = with lib.types; nullOr (strMatching "^([0-9]{1,3}\\.){3}[0-9]{1,3}$");
+      default = null;
+      example = "188.243.124.247";
+      description = ''
+        Optional Hetzner Floating IPv4 configured as `/32` on the primary interface.
+      '';
+    };
    device = lib.mkOption {
      type = lib.types.str;
      default = if isNewer then "/dev/nvme0n1" else "/dev/sda";
@@ -65,14 +78,27 @@ in {
      '';
    };
    networkMatchConfigName = lib.mkOption {
-      type = lib.types.strMatching "^(enp1s0|ens3|eth0)$";
+      type = with lib.types; nullOr str;
+      default = null;
      example = "enp1s0";
      description = ''
-        type of network conection, 
-        on older hetzner servers may be `ens3` or else
-        on newer probably `enp1s0`
+        Optional interface name to match in systemd-networkd.

-        you can use `networkctl list` on server to know it
+        Prefer `networkMatchConfigMac` for stable matching across rescue
+        images and installed systems that may rename interfaces differently.
+
+        You can use `networkctl list` on server to know it.
+      '';
+    };
+    networkMatchConfigMac = lib.mkOption {
+      type = with lib.types; nullOr (strMatching "^([0-9A-Fa-f]{2}:){5}[0-9A-Fa-f]{2}$");
+      default = null;
+      example = "92:00:08:4a:b0:32";
+      description = ''
+        Optional permanent MAC address to match in systemd-networkd.
+
+        This is the preferred Hetzner Cloud matching method because interface
+        names can differ between rescue images and installed NixOS systems.
      '';
    };
  };
@@ -80,6 +106,15 @@ in {
  config = lib.mkIf cfg.enable (lib.mkMerge
  [
    {
+      boot.loader.systemd-boot.enable = false;
+      boot.loader.efi.canTouchEfiVariables = false;
+      boot.loader.grub = {
+        enable = true;
+        efiSupport = true;
+        efiInstallAsRemovable = true;
+        device = "nodev";
+      };
+
      boot.initrd.availableKernelModules = [
        "ata_piix"
        "uhci_hcd"
@@ -90,14 +125,18 @@ in {
      networking.useNetworkd = true;
      systemd.network.enable = true;
      systemd.network.networks."30-wan" = {
-        matchConfig.Name = cfg.networkMatchConfigName;
+        matchConfig = networkMatchConfig;
        networkConfig.DHCP = "no";
        address = [
          "${cfg.ipv4}/32"
          "${cfg.ipv6}::/64"
-        ];
+        ] ++ lib.optional (cfg.floatingIpv4 != null) "${cfg.floatingIpv4}/32";
        routes = [
-          { Gateway = "172.31.1.1"; GatewayOnLink = true; }
+          {
+            Gateway = "172.31.1.1";
+            GatewayOnLink = true;
+            PreferredSource = cfg.ipv4;
+          }
          { Gateway = "fe80::1"; }
        ];
      };
@@ -137,6 +176,13 @@ in {
          };
        };
      };
+
+      assertions = [
+        {
+          assertion = cfg.networkMatchConfigName != null || cfg.networkMatchConfigMac != null;
+          message = "hectic.hardware.hetzner-cloud requires networkMatchConfigName or networkMatchConfigMac";
+        }
+      ];
    } 
    (lib.mkIf (pkgs.stdenv.hostPlatform.system == "aarch64-linux") {
      boot.initrd.kernelModules = [ "virtio_gpu" ];
@@ -0,0 +1,134 @@
+{ ... }:
+{ 
+  lib,
+  config,
+  ...
+}: let
+  cfg = config.hectic.hardware.njalla;
+in {
+  options.hectic.hardware.njalla = {
+    enable = lib.mkEnableOption "Enable njalla hardware configurations";
+    ipv4 = lib.mkOption {
+      type        = lib.types.strMatching "^([0-9]{1,3}\\.){3}[0-9]{1,3}$";
+      example     = "185.193.126.103";
+      description = ''
+        Njalla IPv4 address assigned to the host.
+      '';
+    };
+    ipv4PrefixLength = lib.mkOption {
+      type        = lib.types.int;
+      default     = 24;
+      example     = 24;
+      description = ''
+        Njalla IPv4 prefix length.
+      '';
+    };
+    ipv4Gateway = lib.mkOption {
+      type        = lib.types.strMatching "^([0-9]{1,3}\\.){3}[0-9]{1,3}$";
+      default     = "185.193.126.1";
+      example     = "185.193.126.1";
+      description = ''
+        Njalla IPv4 gateway.
+      '';
+    };
+    ipv6 = lib.mkOption {
+      type        = lib.types.strMatching "^([0-9a-fA-F]{0,4}:){2,7}[0-9a-fA-F]{0,4}$";
+      example     = "2a0a:3840:1337:126:0:b9c1:7e67:1337";
+      description = ''
+        Njalla IPv6 address assigned to the host.
+      '';
+    };
+    ipv6PrefixLength = lib.mkOption {
+      type        = lib.types.int;
+      default     = 64;
+      example     = 64;
+      description = ''
+        Njalla IPv6 prefix length.
+      '';
+    };
+    ipv6Gateway = lib.mkOption {
+      type        = lib.types.strMatching "^([0-9a-fA-F]{0,4}:){2,7}[0-9a-fA-F]{0,4}$";
+      default     = "2a0a:3840:1337:126::1";
+      example     = "2a0a:3840:1337:126::1";
+      description = ''
+        Njalla IPv6 gateway.
+      '';
+    };
+    networkMatchConfigName = lib.mkOption {
+      type        = lib.types.str;
+      default     = "eth0";
+      example     = "eth0";
+      description = ''
+        Njalla container network interface name.
+      '';
+    };
+    device = lib.mkOption {
+      type = lib.types.str;
+      default = "/dev/vda";
+      example = "/dev/disk/by-id/virtio-root";
+      description = ''
+        Njalla installation disk for disko/nixos-anywhere.
+
+        `/dev/vda` is the default block device visible on the inspected Njalla
+        host. Prefer a stable `/dev/disk/by-id/...` path when available.
+      '';
+    };
+    enableDisko = lib.mkOption {
+      type        = lib.types.bool;
+      default     = true;
+      description = ''
+        Whether to provide a disko layout for nixos-anywhere installs.
+      '';
+    };
+  };
+
+  config = lib.mkIf cfg.enable (lib.mkMerge [
+    {
+      boot.isContainer = true;
+
+      networking.useDHCP    = false;
+      networking.useNetworkd = true;
+      systemd.network.enable = true;
+      systemd.network.networks."30-wan" = {
+        matchConfig.Name = cfg.networkMatchConfigName;
+        networkConfig.DHCP = "no";
+        address = [
+          "${cfg.ipv4}/${toString cfg.ipv4PrefixLength}"
+          "${cfg.ipv6}/${toString cfg.ipv6PrefixLength}"
+        ];
+        routes = [
+          { Gateway = cfg.ipv4Gateway; }
+          { Gateway = cfg.ipv6Gateway; }
+        ];
+      };
+
+      networking.nameservers = [ "1.1.1.1" "8.8.8.8" ];
+    }
+    (lib.mkIf cfg.enableDisko {
+      boot.loader.grub.device = cfg.device;
+
+      disko.devices.disk.main = {
+        type = "disk";
+        device = cfg.device;
+        content = {
+          type = "gpt";
+          partitions = {
+            boot = {
+              size     = "1M";
+              type     = "EF02";
+              priority = 1;
+            };
+            root = {
+              size    = "100%";
+              content = {
+                type       = "filesystem";
+                format     = "ext4";
+                mountpoint = "/";
+              };
+            };
+          };
+        };
+      };
+    })
+  ]);
+}
@@ -25,36 +25,7 @@ in {
    programs.bash.shellAliases.tmux = "tmux a";

    home-manager.sharedModules = [
-      {
-        programs.tmux = {
-          enable = true;
-          plugins = with pkgs.tmuxPlugins; [ resurrect continuum ];
-          keyMode = "vi";
-          escapeTime = 500;
-          historyLimit = 50000;
-          newSession = true;
-          extraConfig = ''
-            # resurrect
-            set -g @resurrect-strategy-vim 'session'
-            set -g @resurrect-strategy-nvim 'session'
-            set -g @resurrect-capture-pane-contents 'on'
-
-            resurrect_dir="$HOME/.tmux/resurrect"
-            set -g @resurrect-dir $resurrect_dir
-            set -g @resurrect-hook-post-save-all 'target=$(readlink -f $resurrect_dir/last); sed "s| --cmd .*-vim-pack-dir||g; s|/etc/profiles/per-user/$USER/bin/||g; s|/home/$USER/.nix-profile/bin/||g" $target | sponge $target'
-
-            # continuum
-            set -g @continuum-restore 'on'
-            set -g @continuum-boot 'on'
-            set -g @continuum-save-interval '10'
-
-            bind-key    -T copy-mode-vi v                  send-keys -X begin-selection
-            bind-key    -T copy-mode-vi C-v                send-keys -X rectangle-toggle
-
-            bind-key O select-pane -t :.-
-          '';
-        };
-      }
+      (flake + "/home/module/program/tmux.nix")
    ];

    home-manager.users.root.home.stateVersion = lib.mkDefault "25.05";
@@ -0,0 +1,73 @@
+{ ... }: {
+  lib,
+  config,
+  ...
+}: let
+  cfg = config.hectic.services.attic;
+in {
+  options.hectic.services.attic = {
+    enable = lib.mkEnableOption "Attic binary cache server";
+
+    hostName = lib.mkOption {
+      type = lib.types.str;
+      description = "Public hostname used by clients to reach this Attic server.";
+    };
+
+    listenAddress = lib.mkOption {
+      type = lib.types.str;
+      default = "127.0.0.1";
+      description = "Local address atticd binds to behind the reverse proxy.";
+    };
+
+    port = lib.mkOption {
+      type = lib.types.port;
+      default = 8080;
+      description = "Local port atticd binds to behind the reverse proxy.";
+    };
+
+    environmentFile = lib.mkOption {
+      type = lib.types.path;
+      description = ''
+        SOPS-backed environment file containing Attic JWT and object-storage
+        credentials.
+      '';
+    };
+
+    storage = {
+      bucket = lib.mkOption {
+        type = lib.types.str;
+        description = "Hetzner Object Storage bucket name used by Attic.";
+      };
+
+      endpoint = lib.mkOption {
+        type = lib.types.str;
+        description = "S3-compatible HTTPS endpoint for Hetzner Object Storage.";
+      };
+
+      region = lib.mkOption {
+        type = lib.types.str;
+        description = "Region name for Hetzner Object Storage.";
+      };
+    };
+
+  };
+
+  config = lib.mkIf cfg.enable {
+    services.atticd = {
+      enable          = true;
+      environmentFile = cfg.environmentFile;
+      settings = {
+        listen = "${cfg.listenAddress}:${toString cfg.port}";
+        allowed-hosts = [ cfg.hostName ];
+        api-endpoint = "https://${cfg.hostName}/";
+        compression.type = "zstd";
+        storage = {
+          type     = "s3";
+          bucket   = cfg.storage.bucket;
+          endpoint = cfg.storage.endpoint;
+          region   = cfg.storage.region;
+        };
+      };
+    };
+  };
+}
@@ -0,0 +1,101 @@
+{ ... }: {
+  lib,
+  config,
+  pkgs,
+  ...
+}: let
+  cfg = config.hectic.services.dify;
+
+  difySource = pkgs.fetchFromGitHub {
+    owner = "langgenius";
+    repo = "dify";
+    rev = "00e578606715a9da34488608edee8c68d4ef4893";
+    hash = "sha256-kLxMdmt1FtOl39C9SS7ZGY6C78e3hPiVI5Q3EdbkMPU=";
+  };
+
+  composeOverride = pkgs.writeText "dify-compose.override.yaml" ''
+    services:
+      nginx:
+        ports: !override
+          - "127.0.0.1:${toString cfg.port}:80"
+      plugin_daemon:
+        ports: !override
+          - "127.0.0.1:${toString cfg.pluginPort}:5003"
+        environment:
+          DB_USERNAME: ''${DB_USERNAME:-postgres}
+          DB_HOST: ''${DB_HOST:-db_postgres}
+          DB_PORT: ''${DB_PORT:-5432}
+          REDIS_HOST: ''${REDIS_HOST:-redis}
+          REDIS_PORT: ''${REDIS_PORT:-6379}
+          REDIS_DB: ''${REDIS_DB:-0}
+  '';
+in {
+  options.hectic.services.dify = {
+    enable = lib.mkEnableOption "Dify self-hosted AI platform";
+
+    dataDir = lib.mkOption {
+      type = lib.types.path;
+      default = "/var/lib/dify";
+      description = "Persistent directory for Dify compose state and volumes.";
+    };
+
+    environmentFile = lib.mkOption {
+      type = lib.types.path;
+      description = ''
+        Environment file for Dify. Keep secrets here, including SECRET_KEY,
+        DB_PASSWORD, REDIS_PASSWORD, and plugin daemon credentials.
+      '';
+    };
+
+    port = lib.mkOption {
+      type = lib.types.port;
+      default = 80;
+      description = "Loopback HTTP port forwarded to Dify through SSH.";
+    };
+
+    pluginPort = lib.mkOption {
+      type = lib.types.port;
+      default = 5003;
+      description = "Loopback plugin daemon port forwarded through SSH when needed.";
+    };
+
+    composeProfiles = lib.mkOption {
+      type = lib.types.str;
+      default = "weaviate,postgresql,collaboration";
+      description = "Dify Docker Compose profiles to start.";
+    };
+  };
+
+  config = lib.mkIf cfg.enable {
+    virtualisation.docker.enable = true;
+
+    systemd.services.dify = {
+      description = "Dify Docker Compose stack";
+      wantedBy = [ "multi-user.target" ];
+      wants = [ "docker.service" ];
+      after = [ "docker.service" ];
+      requires = [ "docker.service" ];
+      restartIfChanged = true;
+      serviceConfig = {
+        Type = "oneshot";
+        RemainAfterExit = true;
+        WorkingDirectory = cfg.dataDir;
+        ExecStartPre = pkgs.writeShellScript "dify-prepare" ''
+          set -eu
+          install -d -m 0750 ${lib.escapeShellArg cfg.dataDir}
+          cp -R ${difySource}/docker/. ${lib.escapeShellArg cfg.dataDir}/
+          install -m 0600 ${lib.escapeShellArg cfg.environmentFile} ${lib.escapeShellArg cfg.dataDir}/.env
+        '';
+        ExecStart = "${pkgs.docker-compose}/bin/docker-compose --project-directory ${cfg.dataDir} --file ${cfg.dataDir}/docker-compose.yaml --file ${composeOverride} --env-file ${cfg.dataDir}/.env up --detach";
+        ExecStop = "${pkgs.docker-compose}/bin/docker-compose --project-directory ${cfg.dataDir} --file ${cfg.dataDir}/docker-compose.yaml --file ${composeOverride} --env-file ${cfg.dataDir}/.env down";
+      };
+      environment = {
+        COMPOSE_PROFILES = cfg.composeProfiles;
+      };
+    };
+
+    systemd.tmpfiles.rules = [
+      "d ${cfg.dataDir} 0750 root root -"
+    ];
+  };
+}
@@ -31,7 +31,7 @@ in {

      locations."= /config.element.${matrixDomain}.json".return = "302 /config.json";

-      root = pkgs.element-web.override {
+      root = pkgs.hectic.element-web.override {
        conf = {
          default_server_config = {
            "m.homeserver".base_url = "https://${matrixDomain}";
@@ -43,6 +43,8 @@ in {
            matrixDomain
          ];

+          hectic.videoMessages.enabled = true;
+
          jitsi = lib.optionalAttrs (jitsiPreferredDomain != null) {
            preferred_domain = jitsiPreferredDomain;
          };
@@ -0,0 +1,224 @@
+{ ... }: {
+  lib,
+  config,
+  ...
+}: let
+  cfg = config.hectic.services.ente;
+
+  webHostNames = [
+    cfg.domains.accounts
+    cfg.domains.cast
+    cfg.domains.photos
+  ];
+in {
+  options.hectic.services.ente = {
+    enable = lib.mkEnableOption "Ente Photos self-hosted service";
+
+    apiDomain = lib.mkOption {
+      type = lib.types.str;
+      description = "Public hostname for the Ente Museum API.";
+    };
+
+    domains = {
+      accounts = lib.mkOption {
+        type = lib.types.str;
+        description = "Public hostname for the Ente accounts web app.";
+      };
+
+      cast = lib.mkOption {
+        type = lib.types.str;
+        description = "Public hostname for the Ente cast web app.";
+      };
+
+      albums = lib.mkOption {
+        type = lib.types.str;
+        description = "Public hostname for public Ente album links.";
+      };
+
+      photos = lib.mkOption {
+        type = lib.types.str;
+        description = "Public hostname for the Ente Photos web app.";
+      };
+    };
+
+    maxUploadSize = lib.mkOption {
+      type = lib.types.str;
+      default = "10G";
+      description = "Maximum request body accepted by nginx in front of Museum.";
+    };
+
+    disableRegistration = lib.mkOption {
+      type = lib.types.bool;
+      default = false;
+      description = "Whether Museum should reject new account registration.";
+    };
+
+    smtp = {
+      enable = lib.mkEnableOption "SMTP delivery for Ente verification emails";
+
+      host = lib.mkOption {
+        type = lib.types.str;
+        default = "127.0.0.1";
+        description = "SMTP host Museum uses to send verification emails.";
+      };
+
+      port = lib.mkOption {
+        type = lib.types.port;
+        default = 25;
+        description = "SMTP port Museum uses to send verification emails.";
+      };
+
+      email = lib.mkOption {
+        type = lib.types.str;
+        description = "From email address used by Museum.";
+      };
+
+      senderName = lib.mkOption {
+        type = lib.types.str;
+        default = "Ente Photos";
+        description = "Display name used for Ente verification emails.";
+      };
+
+      encryption = lib.mkOption {
+        type = lib.types.nullOr (lib.types.enum [ "tls" "ssl" ]);
+        default = null;
+        description = "Optional SMTP encryption mode. Leave null for local plaintext SMTP.";
+      };
+    };
+
+    storage = {
+      bucket = lib.mkOption {
+        type = lib.types.str;
+        description = "S3-compatible bucket used by Ente for photo object storage.";
+      };
+
+      endpoint = lib.mkOption {
+        type = lib.types.str;
+        description = "S3-compatible endpoint URL.";
+      };
+
+      region = lib.mkOption {
+        type = lib.types.str;
+        description = "S3-compatible region name.";
+      };
+
+      hotStorage = lib.mkOption {
+        type = lib.types.enum [
+          "b2-eu-cen"
+          "wasabi-eu-central-2-v3"
+          "scw-eu-fr-v3"
+        ];
+        default = "b2-eu-cen";
+        description = ''
+          Museum's primary hot-storage key. Upstream requires one of its
+          historical S3 storage identifiers even when the backing provider is a
+          generic S3-compatible service.
+        '';
+      };
+
+      usePathStyleUrls = lib.mkOption {
+        type = lib.types.bool;
+        default = false;
+        description = "Whether Museum should use path-style S3 URLs.";
+      };
+    };
+
+    secrets = {
+      encryptionKeyFile = lib.mkOption {
+        type = lib.types.path;
+        description = "File containing Museum key.encryption.";
+      };
+
+      hashKeyFile = lib.mkOption {
+        type = lib.types.path;
+        description = "File containing Museum key.hash.";
+      };
+
+      jwtSecretFile = lib.mkOption {
+        type = lib.types.path;
+        description = "File containing Museum jwt.secret.";
+      };
+
+      s3AccessKeyFile = lib.mkOption {
+        type = lib.types.path;
+        description = "File containing the S3 access key.";
+      };
+
+      s3SecretKeyFile = lib.mkOption {
+        type = lib.types.path;
+        description = "File containing the S3 secret key.";
+      };
+    };
+  };
+
+  config = lib.mkIf cfg.enable {
+    services.ente = {
+      api = {
+        enable        = true;
+        enableLocalDB = true;
+        domain        = cfg.apiDomain;
+
+        nginx.enable = true;
+
+        settings = {
+          key = {
+            encryption._secret = cfg.secrets.encryptionKeyFile;
+            hash._secret       = cfg.secrets.hashKeyFile;
+          };
+
+          jwt.secret._secret = cfg.secrets.jwtSecretFile;
+
+          s3 = {
+            hot_storage.primary = cfg.storage.hotStorage;
+            derived-storage     = cfg.storage.hotStorage;
+            are_local_buckets   = false;
+            use_path_style_urls = cfg.storage.usePathStyleUrls;
+
+            ${cfg.storage.hotStorage} = {
+              key._secret    = cfg.secrets.s3AccessKeyFile;
+              secret._secret = cfg.secrets.s3SecretKeyFile;
+              endpoint       = cfg.storage.endpoint;
+              region         = cfg.storage.region;
+              bucket         = cfg.storage.bucket;
+            };
+          };
+
+          internal.disable-registration = cfg.disableRegistration;
+
+          smtp = lib.mkIf cfg.smtp.enable ({
+            inherit (cfg.smtp) host port email;
+            sender-name = cfg.smtp.senderName;
+          } // lib.optionalAttrs (cfg.smtp.encryption != null) {
+            encryption = cfg.smtp.encryption;
+          });
+        };
+      };
+
+      web = {
+        enable = true;
+        domains = {
+          api = cfg.apiDomain;
+          inherit (cfg.domains) accounts cast albums photos;
+        };
+      };
+    };
+
+    services.nginx.virtualHosts =
+      (lib.genAttrs webHostNames (_: {
+        enableACME = true;
+        forceSSL   = true;
+      })) // {
+        ${cfg.apiDomain} = {
+          enableACME = true;
+          forceSSL   = true;
+          extraConfig = lib.mkForce ''
+            client_max_body_size ${cfg.maxUploadSize};
+          '';
+          locations."/".extraConfig = ''
+            proxy_read_timeout 600s;
+            proxy_send_timeout 600s;
+          '';
+        };
+      };
+  };
+}
@@ -0,0 +1,267 @@
+{
+  inputs,
+  flake,
+  self,
+}:
+{
+  pkgs,
+  lib,
+  config,
+  ...
+}:
+let
+  system = pkgs.stdenv.hostPlatform.system;
+  cfg = config.hectic.services.gitea-runner-controller;
+
+  secretPrefix = "gitea-runner-controller";
+  mkControllerSecret = name: {
+    name = "${secretPrefix}/${name}";
+    value = {
+      sopsFile = flake + "/sus/gitea-runners.yaml";
+      key      = "gitea/hectic-lab/controller/${name}";
+    };
+  };
+
+  controllerSecrets = builtins.listToAttrs (map mkControllerSecret [
+    "hcloud-token"
+    "webhook-secret"
+    "admin-token"
+    "ssh-private-key"
+  ]);
+
+  # Registration token is shared with the existing K8s/local runner setup.
+  registrationTokenPath = config.sops.secrets."gitea-runner/org-registration-token".path;
+
+  commonEnvironment =
+    [
+      "GCR_STATE_DIR=/var/lib/gitea-runner-controller"
+      "GCR_GITEA_URL=${cfg.giteaBaseUrl}"
+      "GCR_ALLOWED_REPOS=${lib.concatStringsSep "," cfg.allowedRepos}"
+      "GCR_CONCURRENCY_CAP=${toString cfg.concurrencyCap}"
+      "GCR_PER_REPO_CAP=${toString cfg.perRepoCap}"
+      "GCR_RECONCILE_INTERVAL_SEC=${toString cfg.reconcileIntervalSec}"
+      "GCR_BUDGET_EUR_MONTHLY=${cfg.budgetEurMonthly}"
+      "GCR_HETZNER_LOCATION=${cfg.hetznerLocation}"
+      "HCLOUD_TOKEN_FILE=${config.sops.secrets."${secretPrefix}/hcloud-token".path}"
+      "GITEA_WEBHOOK_SECRET_FILE=${config.sops.secrets."${secretPrefix}/webhook-secret".path}"
+      "GITEA_REGISTRATION_TOKEN_FILE=${registrationTokenPath}"
+      "GITEA_ADMIN_TOKEN_FILE=${config.sops.secrets."${secretPrefix}/admin-token".path}"
+      "GCR_SSH_PRIVKEY_FILE=${config.sops.secrets."${secretPrefix}/ssh-private-key".path}"
+      "GCR_HCLOUD_SSH_KEY_ID=${toString cfg.hcloudSshKeyId}"
+      "GCR_NIX_VERSION=${cfg.nixVersion}"
+      "GCR_NIX_TARBALL_SHA256=${cfg.nixTarballSha256}"
+      "GCR_ARM_NIX_TARBALL_SHA256=${cfg.armNixTarballSha256}"
+      "GCR_ACT_RUNNER_VERSION=${cfg.actRunnerVersion}"
+      "GCR_ACT_RUNNER_SHA256=${cfg.actRunnerSha256}"
+    ]
+    ++ lib.optional (cfg.debugSshPublicKey != null) "\"GCR_DEBUG_SSH_PUBKEY=${cfg.debugSshPublicKey}\""
+    ++ lib.optionals (cfg.imageId != null) [ "GCR_IMAGE_ID=${cfg.imageId}" ]
+    ++ lib.optionals (cfg.armImageId != null) [ "GCR_ARM_IMAGE_ID=${cfg.armImageId}" ]
+    ++ lib.optionals (cfg.nixImageId != null) [ "GCR_NIX_IMAGE_ID=${cfg.nixImageId}" ]
+    ++ lib.optionals (cfg.armNixImageId != null) [ "GCR_ARM_NIX_IMAGE_ID=${cfg.armNixImageId}" ];
+in
+{
+  options = {
+    hectic.services.gitea-runner-controller = {
+      enable = lib.mkEnableOption "gitea-runner-controller — ephemeral Hetzner VM runner controller";
+      listenAddr = lib.mkOption {
+        type = lib.types.str;
+        default = "127.0.0.1";
+        description = "Address the webhook listener binds to.";
+      };
+      listenPort = lib.mkOption {
+        type = lib.types.port;
+        default = 8787;
+        description = "Port the webhook listener binds to.";
+      };
+      webhookHost = lib.mkOption {
+        type = lib.types.str;
+        default = "runners.hectic-lab.com";
+        description = "Public vhost Gitea delivers webhooks to. Requires a DNS A record to this host.";
+      };
+      giteaBaseUrl = lib.mkOption {
+        type = lib.types.str;
+        default = "https://gitea.hectic-lab.com";
+        description = "Public Gitea URL runners register against.";
+      };
+      allowedRepos = lib.mkOption {
+        type = with lib.types; listOf str;
+        default = [ "hectic-lab/util.nix" ];
+        description = "Repos whose workflow_job events may trigger VM creation.";
+      };
+      concurrencyCap = lib.mkOption {
+        type = lib.types.int;
+        default = 2;
+        description = "Maximum simultaneously assigned runner VMs (global); retained idle VMs do not count.";
+      };
+      perRepoCap = lib.mkOption {
+        type = lib.types.int;
+        default = 1;
+        description = "Maximum concurrently assigned runner VMs per repo; retained idle VMs do not count.";
+      };
+      reconcileIntervalSec = lib.mkOption {
+        type = lib.types.int;
+        default = 60;
+        description = "Seconds between reconciliation ticks.";
+      };
+      budgetEurMonthly = lib.mkOption {
+        type = lib.types.str;
+        default = "15";
+        description = "Soft monthly EUR ceiling for estimated VM spend.";
+      };
+      hetznerLocation = lib.mkOption {
+        type = lib.types.str;
+        default = "nbg1";
+        description = "Hetzner location for ephemeral VMs.";
+      };
+      imageId = lib.mkOption {
+        type = with lib.types; nullOr str;
+        default = null;
+        example = "174108912";
+        description = ''
+          Hetzner image/snapshot id for ephemeral VMs (MicroOS base from
+          gitea-runners-build-microos-snapshots). Controller refuses VM
+          creation while null.
+        '';
+      };
+      armImageId = lib.mkOption {
+        type = with lib.types; nullOr str;
+        default = null;
+        example = "423979717";
+        description = ''
+          Hetzner ARM image/snapshot id for ephemeral VMs. Required for labels
+          whose fallback chain includes ARM server types.
+        '';
+      };
+      nixImageId = lib.mkOption {
+        type = with lib.types; nullOr str;
+        default = null;
+        example = "161547269";
+        description = "Hetzner Ubuntu image id used by Nix-capable x86 runners.";
+      };
+      armNixImageId = lib.mkOption {
+        type = with lib.types; nullOr str;
+        default = null;
+        example = "161547270";
+        description = "Hetzner Ubuntu image id used by Nix-capable ARM runners.";
+      };
+      actRunnerVersion = lib.mkOption {
+        type = lib.types.str;
+        default = "1.0.6";
+        description = "gitea-runner release version downloaded at VM bootstrap.";
+      };
+      actRunnerSha256 = lib.mkOption {
+        type = lib.types.str;
+        default = "d78ac11deed6580a2d88c1ef72c522aa7e2986d2d22b0d80edbee577b8f79b20";
+        description = "sha256 of the pinned gitea-runner linux-amd64 binary, verified at bootstrap.";
+      };
+      nixVersion = lib.mkOption {
+        type = lib.types.str;
+        default = "2.35.0";
+        description = "Nix release installed from the official static tarball at bootstrap.";
+      };
+      nixTarballSha256 = lib.mkOption {
+        type = lib.types.str;
+        default = "e828c574555f235f2ca47c4a1432f8a07905d5cf59f087e9581be56c238818db";
+        description = "sha256 of the pinned Nix x86_64-linux tarball, verified at bootstrap.";
+      };
+      armNixTarballSha256 = lib.mkOption {
+        type = lib.types.str;
+        default = "1916bd42cfd4a65d5642a4ca75be4fe55db0a67e5fa16e18808fb5b6d2365bf8";
+        description = "sha256 of the pinned Nix aarch64-linux tarball, verified at bootstrap.";
+      };
+      debugSshPublicKey = lib.mkOption {
+        type = with lib.types; nullOr str;
+        default = null;
+        description = ''
+          Injected into every ephemeral VM (project ssh-key "yukkop@nixos"
+          carries the matching public key; this value is informational and
+          used by gcr_bootstrap_script documentation).
+        '';
+      };
+      hcloudSshKeyId = lib.mkOption {
+        type = lib.types.nullOr lib.types.ints.positive;
+        default = null;
+        description = "Hetzner project SSH key ID injected into ephemeral VMs at creation.";
+      };
+      bootstrapSshPrivateKeyFile = lib.mkOption {
+        type = lib.types.path;
+        description = ''
+          Path (sops-rendered) to the SSH private key the controller uses to
+          push bootstrap into ephemeral VMs. Public half must be registered
+          as Hetzner project ssh-key "yukkop@nixos".
+        '';
+      };
+    };
+  };
+
+  config = lib.mkIf cfg.enable {
+    assertions = [
+      {
+        assertion = cfg.imageId != null;
+        message = "gitea-runner-controller: imageId must be set to a MicroOS snapshot id before enabling";
+      }
+    ];
+
+    sops.secrets = controllerSecrets;
+
+    systemd.services.gitea-runner-controller = {
+      description = "Gitea ephemeral runner controller — reconcile loop";
+      after = [ "network-online.target" ];
+      wants = [ "network-online.target" ];
+      wantedBy = [ "multi-user.target" ];
+      serviceConfig = {
+        Type = "simple";
+        ExecStart = "${self.packages.${system}.gitea-runner-controller}/bin/gitea-runner-controller";
+        Restart = "always";
+        RestartSec = "5s";
+        StateDirectory = "gitea-runner-controller";
+        NoNewPrivileges = true;
+        ProtectSystem = "strict";
+        ProtectHome = true;
+        PrivateTmp = true;
+        StandardOutput = "journal";
+        StandardError = "journal";
+        Environment = commonEnvironment;
+      };
+    };
+
+    systemd.services.gitea-runner-webhook = {
+      description = "Gitea ephemeral runner webhook receiver";
+      after = [
+        "network.target"
+        "gitea-runner-controller.service"
+      ];
+      wantedBy = [ "multi-user.target" ];
+      partOf = [ "gitea-runner-controller.service" ];
+      serviceConfig = {
+        Type = "simple";
+        ExecStart = "${self.packages.${system}.gitea-runner-controller}/bin/gitea-runner-webhook";
+        Restart = "always";
+        RestartSec = "2s";
+        StateDirectory = "gitea-runner-controller";
+        NoNewPrivileges = true;
+        ProtectSystem = "strict";
+        ProtectHome = true;
+        PrivateTmp = true;
+        StandardOutput = "journal";
+        StandardError = "journal";
+        Environment = commonEnvironment ++ [
+          "GCR_LISTEN_ADDR=${cfg.listenAddr}"
+          "GCR_LISTEN_PORT=${toString cfg.listenPort}"
+        ];
+      };
+    };
+
+    services.nginx.virtualHosts."${cfg.webhookHost}" = {
+      enableACME = true;
+      forceSSL = true;
+      locations."/" = {
+        extraConfig = ''
+          proxy_pass http://${cfg.listenAddr}:${toString cfg.listenPort};
+          proxy_read_timeout 30s;
+        '';
+      };
+    };
+  };
+}
@@ -0,0 +1,168 @@
+{ ... }:
+{
+  lib,
+  config,
+  ...
+}:
+let
+  cfg = config.hectic.services.immich;
+in
+{
+  options.hectic.services.immich = {
+    enable = lib.mkEnableOption "Immich self-hosted photo and video service";
+
+    domain = lib.mkOption {
+      type = lib.types.strMatching "[A-Za-z0-9][A-Za-z0-9.-]*";
+      description = "Public hostname used to reach Immich.";
+    };
+
+    mediaLocation = lib.mkOption {
+      type = lib.types.path;
+      default = "/var/lib/immich";
+      description = ''
+        Local filesystem path used for Immich media. Upstream Immich does not
+        support S3 as its media backend; use a local disk or block volume here.
+      '';
+    };
+
+    maxUploadSize = lib.mkOption {
+      type = lib.types.strMatching "[1-9][0-9]*[KMG]?";
+      default = "10G";
+      description = "Maximum request body accepted by nginx in front of Immich.";
+    };
+
+    secretsFile = lib.mkOption {
+      type = lib.types.nullOr (lib.types.strMatching "/[^[:space:]]+");
+      default = null;
+      description = ''
+        SOPS-backed environment file passed to Immich. Use this for secrets
+        such as DB_PASSWORD; never put secret values in Nix configuration.
+      '';
+    };
+
+    machineLearning = {
+      enable = lib.mkOption {
+        type = lib.types.bool;
+        default = true;
+        description = "Whether to enable Immich machine-learning features.";
+      };
+    };
+
+    accelerationDevices = lib.mkOption {
+      type = lib.types.nullOr (lib.types.listOf lib.types.str);
+      default = [ ];
+      description = "Device paths exposed to Immich for hardware acceleration.";
+    };
+
+    storageBox = {
+      enable = lib.mkEnableOption "Hetzner Storage Box media storage";
+
+      host = lib.mkOption {
+        type = lib.types.strMatching "[A-Za-z0-9][A-Za-z0-9.-]*";
+        default = "u666713-sub1.your-storagebox.de";
+        description = "Hetzner Storage Box SMB hostname.";
+      };
+
+      username = lib.mkOption {
+        type = lib.types.strMatching "[A-Za-z0-9][A-Za-z0-9_-]*";
+        default = "u666713-sub1";
+        description = "Storage Box SMB username.";
+      };
+
+      share = lib.mkOption {
+        type = lib.types.strMatching "[A-Za-z0-9][A-Za-z0-9_-]*";
+        default = "u666713-sub1";
+        description = "SMB share exported by Storage Box.";
+      };
+
+      subdirectory = lib.mkOption {
+        type = lib.types.nullOr (lib.types.strMatching "[A-Za-z0-9][A-Za-z0-9_./-]*");
+        default = null;
+        description = "Directory within the SMB share used by Immich.";
+      };
+
+      credentialsFile = lib.mkOption {
+        type = lib.types.nullOr (lib.types.strMatching "/[^[:space:]]+");
+        default = null;
+        description = ''
+          Runtime file containing the SMB password. Keep this in a SOPS
+          secret, outside the Nix store.
+        '';
+      };
+    };
+  };
+
+  config = lib.mkIf cfg.enable {
+    assertions = [
+      {
+        assertion = cfg.secretsFile == null || !lib.hasPrefix "/nix/store/" cfg.secretsFile;
+        message = "hectic.services.immich.secretsFile must reference a runtime secret path, not /nix/store.";
+      }
+      {
+        assertion = !cfg.storageBox.enable || cfg.storageBox.credentialsFile != null;
+        message = "hectic.services.immich.storageBox.credentialsFile is required when Storage Box is enabled.";
+      }
+      {
+        assertion =
+          !cfg.storageBox.enable
+          || cfg.storageBox.credentialsFile == null
+          || !lib.hasPrefix "/nix/store/" cfg.storageBox.credentialsFile;
+        message = "hectic.services.immich.storageBox.credentialsFile must reference a runtime secret path, not /nix/store.";
+      }
+    ];
+
+    services.immich = {
+      enable = true;
+      host = "127.0.0.1";
+      mediaLocation = cfg.mediaLocation;
+      secretsFile = cfg.secretsFile;
+      accelerationDevices = cfg.accelerationDevices;
+      machine-learning.enable = cfg.machineLearning.enable;
+      settings.server.externalDomain = "https://${cfg.domain}";
+    };
+
+    services.nginx = {
+      enable = true;
+      virtualHosts.${cfg.domain} = {
+        enableACME = true;
+        forceSSL = true;
+        extraConfig = lib.mkForce ''
+          client_max_body_size ${cfg.maxUploadSize};
+        '';
+        locations."/" = {
+          proxyPass = "http://127.0.0.1:${toString config.services.immich.port}";
+          proxyWebsockets = true;
+          extraConfig = ''
+            proxy_read_timeout 600s;
+            proxy_send_timeout 600s;
+          '';
+        };
+      };
+    };
+
+    fileSystems.${toString cfg.mediaLocation} = lib.mkIf cfg.storageBox.enable {
+      device = "//${cfg.storageBox.host}/${cfg.storageBox.share}";
+      fsType = "cifs";
+      options = [
+        "_netdev"
+        "nofail"
+        "x-systemd.mount-timeout=60s"
+        "vers=3.1.1"
+        "seal"
+        "cache=none"
+        "credentials=${cfg.storageBox.credentialsFile}"
+        "username=${cfg.storageBox.username}"
+        "uid=${config.services.immich.user}"
+        "gid=${config.services.immich.group}"
+        "file_mode=0660"
+        "dir_mode=0770"
+      ] ++ lib.optional (cfg.storageBox.subdirectory != null)
+        "prefixpath=${cfg.storageBox.subdirectory}";
+    };
+
+    systemd.services.immich-server.unitConfig.RequiresMountsFor = lib.mkIf cfg.storageBox.enable [
+      cfg.mediaLocation
+    ];
+    systemd.services.immich-server.serviceConfig.Restart = lib.mkForce "always";
+  };
+}
@@ -56,8 +56,18 @@ in {
      certificateScheme = "acme-nginx";
    };

-    # NOTE(yukkop): avoid Gmail rejection due to missing IPv6 PTR records
-    services.postfix.settings.main.inet_protocols = lib.mkDefault "ipv4";
+    services.postfix.settings.main = {
+      # NOTE(yukkop): avoid Gmail rejection due to missing IPv6 PTR records.
+      inet_protocols = lib.mkDefault "ipv4";
+
+      # NOTE(yukkop): nixos-mailserver enables DANE by default. Some large MXes
+      # currently fail certificate verification under this policy, which leaves
+      # otherwise valid transactional mail deferred in the queue. Keep STARTTLS
+      # opportunistic for outbound delivery rather than blocking mail entirely.
+      smtp_tls_security_level = lib.mkForce "may";
+      smtp_dns_support_level  = lib.mkForce "enabled";
+      smtp_tls_policy_maps    = lib.mkForce "";
+    };

    security.acme.acceptTerms       = true;
    security.acme.defaults.email    = "security@" + cfg.domain;
@@ -0,0 +1,356 @@
+{ ... }:
+{
+  pkgs,
+  lib,
+  config,
+  ...
+}: let
+  cfg = config.hectic.services.p4d;
+
+  boolString = value: if value then "1" else "0";
+
+  serviceUser = cfg.user;
+  serviceGroup = cfg.group;
+  packageExe = if cfg.package != null then "${cfg.package}/bin/p4d" else "/run/current-system/sw/bin/false";
+  clientExe = if cfg.clientPackage != null then "${cfg.clientPackage}/bin/p4" else "/run/current-system/sw/bin/false";
+
+  journalFile = "${cfg.journalDir}/journal";
+  logFile = "${cfg.dataDir}/logs/p4d.log";
+  sslPort = if cfg.ssl.enable then "ssl:${cfg.listenAddress}:${toString cfg.port}" else "${cfg.listenAddress}:${toString cfg.port}";
+  bootstrapPort = "${cfg.bootstrap.listenAddress}:${toString cfg.bootstrap.port}";
+
+  typemapLines = [
+    "binary+l //....uasset"
+    "binary+l //....umap"
+    "binary+l //....upk"
+    "binary+l //....udk"
+    "binary+l //....ubulk"
+    "binary+l //....uexp"
+    "binary+l //....utoc"
+    "binary+l //....ucas"
+    "binary+w //....exe"
+    "binary+w //....dll"
+    "binary+w //....lib"
+    "binary+w //....app"
+    "binary+w //....dylib"
+    "binary+w //....stub"
+    "binary+w //....ipa"
+    "binary+w //....pdb"
+  ] ++ cfg.typemap.extraLines;
+
+  typemapSpec = pkgs.writeText "p4d-unreal-typemap.txt" (
+    lib.concatStringsSep "\n" ([ "Typemap:" ] ++ map (line: "\t${line}") typemapLines) + "\n"
+  );
+
+  mainArgs = [
+    "-r" cfg.dataDir
+    "-p" sslPort
+    "-J" journalFile
+    "-L" logFile
+  ] ++ lib.optional (!cfg.caseSensitive) "-C1";
+
+  bootstrapArgs = [
+    "-r" cfg.dataDir
+    "-p" bootstrapPort
+    "-J" journalFile
+    "-L" "${cfg.dataDir}/logs/bootstrap.log"
+  ] ++ lib.optional (!cfg.caseSensitive) "-C1";
+
+  initScript = ''
+    set -eu
+
+    export P4ROOT=${lib.escapeShellArg cfg.dataDir}
+    export P4JOURNAL=${lib.escapeShellArg journalFile}
+    export P4SSLDIR=${lib.escapeShellArg cfg.ssl.dir}
+
+    marker_file="$P4ROOT/.hectic-p4d-initialized"
+    typemap_target="$P4ROOT/unreal-engine.typemap"
+    server_id_file="$P4ROOT/server.id"
+
+    ${pkgs.coreutils}/bin/install -d -m 0750 "$P4ROOT"
+    ${pkgs.coreutils}/bin/install -d -m 0750 ${lib.escapeShellArg cfg.journalDir}
+    ${pkgs.coreutils}/bin/install -d -m 0750 ${lib.escapeShellArg cfg.checkpointDir}
+    ${pkgs.coreutils}/bin/install -d -m 0750 "$P4ROOT/logs"
+    ${pkgs.coreutils}/bin/install -d -m 0700 ${lib.escapeShellArg cfg.ssl.dir}
+
+    if [ ! -f "$server_id_file" ]; then
+      umask 077
+      ${pkgs.coreutils}/bin/printf '%s\n' ${lib.escapeShellArg cfg.serverId} > "$server_id_file"
+    fi
+
+    if [ "${boolString cfg.typemap.enable}" = "1" ]; then
+      ${pkgs.coreutils}/bin/install -m 0640 ${lib.escapeShellArg typemapSpec} "$typemap_target"
+    fi
+
+    if [ "${boolString cfg.ssl.enable}" = "1" ] && {
+      [ ! -f "$P4SSLDIR/privatekey.txt" ] || [ ! -f "$P4SSLDIR/certificate.txt" ];
+    }; then
+      ${packageExe} -r "$P4ROOT" -Gc
+    fi
+
+    if [ ! -f "$marker_file" ] && [ "${boolString cfg.unicode}" = "1" ] && [ ! -e "$P4ROOT/db.counters" ]; then
+      ${packageExe} ${lib.escapeShellArgs ([ "-r" cfg.dataDir ] ++ lib.optional (!cfg.caseSensitive) "-C1" ++ [ "-xi" ])}
+    fi
+
+    if [ ! -f "$marker_file" ] && [ "${boolString cfg.bootstrap.enable}" = "1" ]; then
+      temp_pid_file="$P4ROOT/bootstrap.pid"
+
+      cleanup() {
+        if [ -f "$temp_pid_file" ]; then
+          pid=$(cat "$temp_pid_file")
+          kill "$pid" 2>/dev/null || true
+          wait "$pid" 2>/dev/null || true
+          rm -f "$temp_pid_file"
+        fi
+      }
+
+      trap cleanup EXIT INT TERM
+
+      ${packageExe} ${lib.escapeShellArgs bootstrapArgs} &
+      pid=$!
+      ${pkgs.coreutils}/bin/printf '%s\n' "$pid" > "$temp_pid_file"
+
+      ready=0
+      tries=0
+      while [ "$tries" -lt 30 ]; do
+        if ${clientExe} -p ${lib.escapeShellArg bootstrapPort} info >/dev/null 2>&1; then
+          ready=1
+          break
+        fi
+        tries=$((tries + 1))
+        sleep 1
+      done
+
+      if [ "$ready" -ne 1 ]; then
+        printf 'p4d bootstrap server failed to become ready on %s\n' ${lib.escapeShellArg bootstrapPort} >&2
+        exit 1
+      fi
+
+      ${lib.optionalString (cfg.securityLevel != null) ''
+        ${clientExe} -p ${lib.escapeShellArg bootstrapPort} configure set security=${toString cfg.securityLevel}
+      ''}
+
+      if [ "${boolString cfg.typemap.enable}" = "1" ]; then
+        ${clientExe} -p ${lib.escapeShellArg bootstrapPort} typemap -i < "$typemap_target"
+      fi
+
+      ${clientExe} -p ${lib.escapeShellArg bootstrapPort} admin stop || true
+      wait "$pid" 2>/dev/null || true
+      rm -f "$temp_pid_file"
+      trap - EXIT INT TERM
+    fi
+
+    if [ ! -f "$marker_file" ]; then
+      : > "$marker_file"
+    fi
+  '';
+in {
+  options.hectic.services.p4d = {
+    enable = lib.mkEnableOption "Perforce Helix Core p4d server";
+
+    package = lib.mkOption {
+      type = with lib.types; nullOr package;
+      default = null;
+      defaultText = lib.literalExpression "pkgs.p4d";
+      description = ''
+        Package providing the `p4d` executable. Left null by default because
+        nixpkgs marks Perforce packages unfree.
+      '';
+    };
+
+    clientPackage = lib.mkOption {
+      type = with lib.types; nullOr package;
+      default = null;
+      defaultText = lib.literalExpression "pkgs.p4";
+      description = ''
+        Package providing the `p4` client executable used for optional local
+        bootstrap tasks like typemap installation and security configuration.
+      '';
+    };
+
+    user = lib.mkOption {
+      type = lib.types.str;
+      default = "p4d";
+      description = "System user running the p4d service.";
+    };
+
+    group = lib.mkOption {
+      type = lib.types.str;
+      default = "p4d";
+      description = "System group running the p4d service.";
+    };
+
+    dataDir = lib.mkOption {
+      type = lib.types.str;
+      default = "/var/lib/p4d";
+      description = "Persistent P4ROOT directory for metadata and archives.";
+    };
+
+    journalDir = lib.mkOption {
+      type = lib.types.str;
+      default = "/var/lib/p4d/journal";
+      description = "Directory storing the live journal file.";
+    };
+
+    checkpointDir = lib.mkOption {
+      type = lib.types.str;
+      default = "/var/lib/p4d/checkpoints";
+      description = "Directory reserved for checkpoints and offline backups.";
+    };
+
+    listenAddress = lib.mkOption {
+      type = lib.types.str;
+      default = "0.0.0.0";
+      description = "Address p4d listens on for normal client traffic.";
+    };
+
+    port = lib.mkOption {
+      type = lib.types.port;
+      default = 1666;
+      description = "TCP port exposed by p4d.";
+    };
+
+    openFirewall = lib.mkOption {
+      type = lib.types.bool;
+      default = false;
+      description = "Whether to open the p4d TCP port in the firewall.";
+    };
+
+    serverId = lib.mkOption {
+      type = lib.types.str;
+      default = "master";
+      description = "Perforce server identifier written to P4ROOT/server.id.";
+    };
+
+    unicode = lib.mkOption {
+      type = lib.types.bool;
+      default = true;
+      description = "Initialize Unicode mode on first boot. Irreversible after init.";
+    };
+
+    caseSensitive = lib.mkOption {
+      type = lib.types.bool;
+      default = true;
+      description = ''
+        Whether the server remains case-sensitive. Linux default stays true.
+        Set false only when tooling requires case-insensitive behavior.
+      '';
+    };
+
+    securityLevel = lib.mkOption {
+      type = with lib.types; nullOr (ints.between 0 5);
+      default = 4;
+      description = ''
+        Optional `p4 configure set security=<level>` value applied during local
+        bootstrap. Set null to skip automatic security tuning.
+      '';
+    };
+
+    ssl = {
+      enable = lib.mkOption {
+        type = lib.types.bool;
+        default = true;
+        description = "Whether p4d listens with SSL enabled.";
+      };
+
+      dir = lib.mkOption {
+        type = lib.types.str;
+        default = "/var/lib/p4d/ssl";
+        description = "Directory holding p4d SSL certificate and key material.";
+      };
+    };
+
+    bootstrap = {
+      enable = lib.mkOption {
+        type = lib.types.bool;
+        default = false;
+        description = ''
+          Run one-time local bootstrap tasks on first start: optional security
+          level configuration and Unreal typemap installation.
+        '';
+      };
+
+      listenAddress = lib.mkOption {
+        type = lib.types.str;
+        default = "127.0.0.1";
+        description = "Loopback address used by the local temporary bootstrap server.";
+      };
+
+      port = lib.mkOption {
+        type = lib.types.port;
+        default = 1667;
+        description = "Loopback-only port used by the temporary bootstrap server.";
+      };
+    };
+
+    typemap = {
+      enable = lib.mkOption {
+        type = lib.types.bool;
+        default = true;
+        description = ''
+          Generate the Unreal-focused typemap spec and optionally install it
+          during bootstrap.
+        '';
+      };
+
+      extraLines = lib.mkOption {
+        type = with lib.types; listOf str;
+        default = [ ];
+        description = "Extra lines appended to the generated Perforce typemap spec.";
+      };
+    };
+  };
+
+  config = lib.mkIf cfg.enable {
+    assertions = [
+      {
+        assertion = cfg.package != null;
+        message = "hectic.services.p4d.package must be set, for example pkgs.p4d.";
+      }
+      {
+        assertion = !cfg.bootstrap.enable || cfg.clientPackage != null;
+        message = "hectic.services.p4d.clientPackage must be set when bootstrap.enable is true.";
+      }
+    ];
+
+    users.groups.${serviceGroup} = { };
+    users.users.${serviceUser} = {
+      isSystemUser = true;
+      group = serviceGroup;
+      home = cfg.dataDir;
+      createHome = true;
+    };
+
+    systemd.tmpfiles.rules = [
+      "d ${cfg.dataDir} 0750 ${serviceUser} ${serviceGroup} - -"
+      "d ${cfg.journalDir} 0750 ${serviceUser} ${serviceGroup} - -"
+      "d ${cfg.checkpointDir} 0750 ${serviceUser} ${serviceGroup} - -"
+      "d ${cfg.dataDir}/logs 0750 ${serviceUser} ${serviceGroup} - -"
+      "d ${cfg.ssl.dir} 0700 ${serviceUser} ${serviceGroup} - -"
+    ];
+
+    systemd.services.p4d = {
+      description = "Perforce Helix Core p4d server";
+      after = [ "network-online.target" ];
+      wants = [ "network-online.target" ];
+      wantedBy = [ "multi-user.target" ];
+      preStart = initScript;
+      serviceConfig = {
+        Type = "simple";
+        User = serviceUser;
+        Group = serviceGroup;
+        WorkingDirectory = cfg.dataDir;
+        Environment = lib.mkIf cfg.ssl.enable "P4SSLDIR=${cfg.ssl.dir}";
+        ExecStart = "${packageExe} ${lib.escapeShellArgs mainArgs}";
+        Restart = "on-failure";
+        RestartSec = "5s";
+        TimeoutStopSec = "30s";
+        KillSignal = "SIGTERM";
+        KillMode = "mixed";
+        UMask = "0077";
+      };
+    };
+
+    networking.firewall.allowedTCPPorts = lib.mkIf cfg.openFirewall [ cfg.port ];
+  };
+}
@@ -0,0 +1,568 @@
+{ ... }:
+{
+  pkgs,
+  lib,
+  config,
+  ...
+}: let
+  cfg = config.hectic.services."project-zomboid";
+  serverProperties = cfg.serverProperties // {
+    DefaultPort = cfg.port;
+    UDPPort = cfg.udpPort;
+    WorkshopItems = lib.concatStringsSep ";" cfg.workshopItems;
+    Mods = lib.concatStringsSep ";" cfg.mods;
+  };
+  configLines = lib.mapAttrsToList (
+    name: value:
+      "${name}=${if builtins.isBool value then lib.boolToString value else toString value}"
+  ) serverProperties;
+  sandboxValueType = lib.types.oneOf [
+    lib.types.bool
+    lib.types.int
+    lib.types.float
+    lib.types.str
+    (lib.types.attrsOf sandboxValueType)
+  ];
+  luaValue = value:
+    if builtins.isBool value then
+      lib.boolToString value
+    else if builtins.isInt value || builtins.isFloat value then
+      toString value
+    else if builtins.isAttrs value then
+      "{ ${lib.concatStringsSep " " (lib.mapAttrsToList (name: child: "[${luaValue name}] = ${luaValue child},") value)} }"
+    else
+      "\"${lib.replaceStrings [ "\\" "\"" "\n" "\r" ] [ "\\\\" "\\\"" "\\n" "\\r" ] value}\"";
+  sandboxConfigLines = lib.mapAttrsToList (
+    name: value: "[${luaValue name}] = ${luaValue value},"
+  ) cfg.sandboxProperties;
+  zomboidDir = "${cfg.dataDir}/Zomboid";
+  adminPasswordFile = "${cfg.dataDir}/admin-password";
+  rconPasswordFile = cfg.rcon.passwordFile;
+  backupCfg = cfg.backup;
+  s3CredentialsFile = if backupCfg.s3.credentialsFile == null then "" else backupCfg.s3.credentialsFile;
+  s3Bucket = if backupCfg.s3.bucket == null then "" else backupCfg.s3.bucket;
+  s3Endpoint = if backupCfg.s3.endpoint == null then "" else backupCfg.s3.endpoint;
+  s3Region = if backupCfg.s3.region == null then "" else backupCfg.s3.region;
+  saveDir = "${zomboidDir}/Saves/Multiplayer/${cfg.serverName}";
+  serverConfigDir = "${zomboidDir}/Server";
+  backupScript = pkgs.writeShellScript "project-zomboid-backup" ''
+    set -eu
+
+    staging_dir=${lib.escapeShellArg backupCfg.stagingDir}
+    archive_dir=${lib.escapeShellArg backupCfg.archiveDir}
+    lock_file="$archive_dir/.backup.lock"
+
+    ${pkgs.coreutils}/bin/install -d -m 0700 \
+      "$staging_dir/Zomboid/Saves/Multiplayer/${cfg.serverName}" \
+      "$staging_dir/Zomboid/Server" \
+      "$archive_dir"
+
+    exec 9>"$lock_file"
+    if ! ${pkgs.util-linux}/bin/flock -n 9; then
+      ${pkgs.coreutils}/bin/printf '%s\n' 'Project Zomboid backup already running; skipping.' >&2
+      exit 0
+    fi
+
+    ${lib.optionalString cfg.rcon.enable ''
+      rcon_password="$(${pkgs.coreutils}/bin/cat ${lib.escapeShellArg rconPasswordFile})"
+      if [ -z "$rcon_password" ]; then
+        ${pkgs.coreutils}/bin/printf '%s\n' 'Project Zomboid RCON password file is empty.' >&2
+        exit 1
+      fi
+      ${pkgs.rcon}/bin/rcon \
+        --host 127.0.0.1 \
+        --port ${toString cfg.rcon.port} \
+        --password "$rcon_password" \
+        save
+      ${pkgs.coreutils}/bin/sleep ${toString backupCfg.saveWaitSeconds}
+    ''}
+
+    sync_staging() {
+      ${pkgs.rsync}/bin/rsync -a --delete \
+        ${lib.escapeShellArg "${saveDir}/"} \
+        "$staging_dir/Zomboid/Saves/Multiplayer/${cfg.serverName}/"
+      ${pkgs.rsync}/bin/rsync -a --delete --delete-excluded \
+        --include=${lib.escapeShellArg "/${cfg.serverName}_SandboxVars.lua"} \
+        --include=${lib.escapeShellArg "/${cfg.serverName}_spawnpoints.lua"} \
+        --include=${lib.escapeShellArg "/${cfg.serverName}_spawnregions.lua"} \
+        --exclude='*' \
+        ${lib.escapeShellArg "${serverConfigDir}/"} \
+        "$staging_dir/Zomboid/Server/"
+    }
+
+    # Second pass narrows, but cannot eliminate, live-save inconsistency.
+    sync_staging
+    ${pkgs.coreutils}/bin/sleep 5
+    sync_staging
+
+    timestamp="$(${pkgs.coreutils}/bin/date -u +%Y%m%dT%H%M%SZ)"
+    archive_name="project-zomboid-${cfg.serverName}-$timestamp.tar.zst"
+    archive_tmp="$archive_dir/.$archive_name.tmp"
+    archive="$archive_dir/$archive_name"
+    trap '${pkgs.coreutils}/bin/rm -f "$archive_tmp"' EXIT
+    ${pkgs.gnutar}/bin/tar \
+      --use-compress-program=${lib.escapeShellArg "${pkgs.zstd}/bin/zstd -T0"} \
+      -C "$staging_dir" -cf "$archive_tmp" Zomboid
+    ${pkgs.coreutils}/bin/chmod 0600 "$archive_tmp"
+    ${pkgs.coreutils}/bin/mv "$archive_tmp" "$archive"
+    trap - EXIT
+
+    ${pkgs.findutils}/bin/find "$archive_dir" -maxdepth 1 -type f \
+      -name ${lib.escapeShellArg "project-zomboid-${cfg.serverName}-*.tar.zst"} \
+      -mmin +${toString (backupCfg.retentionDays * 1440)} -delete
+
+    ${lib.optionalString backupCfg.s3.enable ''
+      if [ -z "''${AWS_ACCESS_KEY_ID:-}" ] || [ -z "''${AWS_SECRET_ACCESS_KEY:-}" ]; then
+        ${pkgs.coreutils}/bin/printf '%s\n' \
+          'AWS_ACCESS_KEY_ID or AWS_SECRET_ACCESS_KEY missing from Project Zomboid S3 credentials file.' >&2
+        exit 1
+      fi
+      s3_bucket=${lib.escapeShellArg s3Bucket}
+      s3_prefix=${lib.escapeShellArg backupCfg.s3.prefix}
+      s3_key="''${s3_prefix:+$s3_prefix/}$archive_name"
+      ${pkgs.awscli2}/bin/aws s3 cp "$archive" \
+        "s3://$s3_bucket/$s3_key" \
+        --endpoint-url ${lib.escapeShellArg s3Endpoint} \
+        --region ${lib.escapeShellArg s3Region} \
+        --cli-connect-timeout 30 \
+        --cli-read-timeout 300 \
+        --only-show-errors
+
+      remote_prefix="$s3_prefix"
+      if [ -n "$remote_prefix" ]; then
+        remote_prefix="$remote_prefix/"
+      fi
+      archive_prefix=${lib.escapeShellArg "project-zomboid-${cfg.serverName}-"}
+      remote_list="$staging_dir/.remote-objects.json"
+      remote_delete_dir="$staging_dir/.remote-delete"
+      ${pkgs.awscli2}/bin/aws s3api list-objects-v2 \
+        --bucket "$s3_bucket" \
+        --prefix "$remote_prefix" \
+        --endpoint-url ${lib.escapeShellArg s3Endpoint} \
+        --region ${lib.escapeShellArg s3Region} \
+        --output json > "$remote_list"
+      ${pkgs.python3}/bin/python3 - "$remote_list" "$remote_delete_dir" \
+        "$(( $(${pkgs.coreutils}/bin/date +%s) - ${toString (backupCfg.s3.remoteRetentionDays * 86400)} ))" \
+        "$remote_prefix$archive_prefix" <<'PY'
+import datetime
+import json
+import os
+import re
+import sys
+
+objects_path, delete_dir, cutoff, key_prefix = sys.argv[1:]
+cutoff = int(cutoff)
+archive_pattern = re.compile(
+    re.escape(key_prefix) + r"\d{8}T\d{6}Z\.tar\.zst\Z"
+)
+with open(objects_path, encoding="utf-8") as stream:
+    objects = json.load(stream).get("Contents", [])
+
+old_keys = []
+for item in objects:
+    key = item.get("Key", "")
+    if not archive_pattern.fullmatch(key):
+        continue
+    modified = datetime.datetime.fromisoformat(
+        item["LastModified"].replace("Z", "+00:00")
+    )
+    if int(modified.timestamp()) < cutoff:
+        old_keys.append(key)
+
+os.makedirs(delete_dir, exist_ok=True)
+for batch_number in range(0, len(old_keys), 1000):
+    batch = old_keys[batch_number:batch_number + 1000]
+    manifest_path = os.path.join(
+        delete_dir, f"batch-{batch_number // 1000:04d}.json"
+    )
+    with open(manifest_path, "w", encoding="utf-8") as stream:
+        json.dump(
+            {"Objects": [{"Key": key} for key in batch], "Quiet": True},
+            stream,
+        )
+PY
+      for remote_manifest in "$remote_delete_dir"/*.json; do
+        [ -f "$remote_manifest" ] || continue
+        ${pkgs.awscli2}/bin/aws s3api delete-objects \
+          --bucket "$s3_bucket" \
+          --delete "file://$remote_manifest" \
+          --endpoint-url ${lib.escapeShellArg s3Endpoint} \
+          --region ${lib.escapeShellArg s3Region} \
+          --only-show-errors
+      done
+      ${pkgs.coreutils}/bin/rm -rf "$remote_list" "$remote_delete_dir"
+    ''}
+  '';
+  startScript = pkgs.writeShellScript "project-zomboid-start" ''
+    admin_password=$(${pkgs.coreutils}/bin/cat ${lib.escapeShellArg adminPasswordFile})
+    exec ${pkgs.steam-run}/bin/steam-run \
+      ${lib.escapeShellArg "${cfg.installDir}/start-server.sh"} \
+      -servername ${lib.escapeShellArg cfg.serverName} \
+      -adminpassword "$admin_password"
+  '';
+in {
+  options.hectic.services."project-zomboid" = {
+    enable = lib.mkEnableOption "Project Zomboid dedicated server";
+
+    serverName = lib.mkOption {
+      type = lib.types.strMatching "[A-Za-z0-9._-]+";
+      default = "servertest";
+      description = "Name used for Project Zomboid server and save files.";
+    };
+
+    dataDir = lib.mkOption {
+      type = lib.types.path;
+      default = "/var/lib/project-zomboid";
+      description = "Writable state directory for the server.";
+    };
+
+    installDir = lib.mkOption {
+      type = lib.types.path;
+      default = "/var/lib/project-zomboid/server";
+      description = "Directory where SteamCMD installs the dedicated server.";
+    };
+
+    port = lib.mkOption {
+      type = lib.types.port;
+      default = 16261;
+      description = "Project Zomboid UDP discovery port.";
+    };
+
+    udpPort = lib.mkOption {
+      type = lib.types.port;
+      default = 16262;
+      description = "Project Zomboid UDP game port.";
+    };
+
+    branch = lib.mkOption {
+      type = lib.types.nullOr lib.types.str;
+      default = null;
+      description = "Steam beta branch, for example legacy41.";
+    };
+
+    memory = lib.mkOption {
+      type = lib.types.strMatching "[0-9]+[mMgG]";
+      default = "3g";
+      description = "Maximum Java heap for the server, for example 3g.";
+    };
+
+    workshopItems = lib.mkOption {
+      type = lib.types.listOf lib.types.str;
+      default = [ ];
+      description = "Steam Workshop item IDs, downloaded and enabled by the server.";
+    };
+
+    mods = lib.mkOption {
+      type = lib.types.listOf lib.types.str;
+      default = [ ];
+      description = "Project Zomboid mod loading IDs from mod.info.";
+    };
+
+    serverProperties = lib.mkOption {
+      type = lib.types.attrsOf (
+        lib.types.oneOf [
+          lib.types.bool
+          lib.types.int
+          lib.types.str
+        ]
+      );
+      default = { };
+      description = "Additional or overriding values for the server INI file.";
+    };
+
+    serverPropertiesFile = lib.mkOption {
+      type = lib.types.nullOr lib.types.path;
+      default = null;
+      description = "Runtime file with additional INI values, suitable for secrets.";
+    };
+
+    sandboxProperties = lib.mkOption {
+      type = lib.types.attrsOf sandboxValueType;
+      default = { };
+      description = "Values for the Project Zomboid SandboxVars.lua file.";
+    };
+
+    openFirewall = lib.mkOption {
+      type = lib.types.bool;
+      default = true;
+      description = "Open the Project Zomboid UDP ports in the firewall.";
+    };
+
+    rcon = {
+      enable = lib.mkEnableOption "local RCON for Project Zomboid automation";
+
+      port = lib.mkOption {
+        type = lib.types.port;
+        default = 27015;
+        description = "RCON TCP port; not opened in the firewall by this module.";
+      };
+
+      passwordFile = lib.mkOption {
+        type = lib.types.path;
+        default = "${cfg.dataDir}/rcon-password";
+        description = "Runtime file containing the generated RCON password.";
+      };
+    };
+
+    backup = {
+      enable = lib.mkEnableOption "no-stop Project Zomboid backups";
+
+      onCalendar = lib.mkOption {
+        type = lib.types.str;
+        default = "*:0/30";
+        description = "systemd calendar expression controlling backup frequency.";
+      };
+
+      stagingDir = lib.mkOption {
+        type = lib.types.path;
+        default = "${cfg.dataDir}/backups/staging";
+        description = "Local directory containing the two-pass rsync staging tree.";
+      };
+
+      archiveDir = lib.mkOption {
+        type = lib.types.path;
+        default = "${cfg.dataDir}/backups/archive";
+        description = "Local directory containing timestamped tar.zst archives.";
+      };
+
+      retentionDays = lib.mkOption {
+        type = lib.types.ints.positive;
+        default = 14;
+        description = "Delete local archives older than this many days.";
+      };
+
+      saveWaitSeconds = lib.mkOption {
+        type = lib.types.ints.positive;
+        default = 10;
+        description = "Seconds to wait after the RCON save command before rsync.";
+      };
+
+      s3 = {
+        enable = lib.mkEnableOption "uploading Project Zomboid backups to S3-compatible storage";
+
+        credentialsFile = lib.mkOption {
+          type = lib.types.nullOr lib.types.str;
+          default = null;
+          description = ''
+            Runtime env file containing AWS_ACCESS_KEY_ID and
+            AWS_SECRET_ACCESS_KEY. Required when S3 upload is enabled.
+          '';
+        };
+
+        bucket = lib.mkOption {
+          type = lib.types.nullOr lib.types.str;
+          default = null;
+          description = "S3 bucket receiving backup archives.";
+        };
+
+        endpoint = lib.mkOption {
+          type = lib.types.nullOr lib.types.str;
+          default = null;
+          description = "S3-compatible endpoint URL.";
+        };
+
+        region = lib.mkOption {
+          type = lib.types.nullOr lib.types.str;
+          default = null;
+          description = "S3 region passed to awscli2.";
+        };
+
+        prefix = lib.mkOption {
+          type = lib.types.str;
+          default = "project-zomboid";
+          description = "Optional object key prefix within the S3 bucket.";
+        };
+
+        remoteRetentionDays = lib.mkOption {
+          type = lib.types.ints.positive;
+          default = 14;
+          description = "Delete uploaded archives older than this many days.";
+        };
+      };
+    };
+  };
+
+  config = lib.mkIf cfg.enable {
+    assertions = [
+      {
+        assertion = !cfg.rcon.enable || (
+          lib.hasPrefix "/" cfg.rcon.passwordFile
+          && !lib.hasPrefix "/nix/store/" cfg.rcon.passwordFile
+        );
+        message = "hectic.services.project-zomboid.rcon.passwordFile must be a runtime path outside /nix/store.";
+      }
+      {
+        assertion = !backupCfg.s3.enable || backupCfg.enable;
+        message = "hectic.services.project-zomboid.backup must be enabled before S3 upload.";
+      }
+      {
+        assertion = !backupCfg.s3.enable || backupCfg.s3.credentialsFile != null;
+        message = "hectic.services.project-zomboid.backup.s3.credentialsFile is required when S3 upload is enabled.";
+      }
+      {
+        assertion = !backupCfg.s3.enable || backupCfg.s3.bucket != null;
+        message = "hectic.services.project-zomboid.backup.s3.bucket is required when S3 upload is enabled.";
+      }
+      {
+        assertion = !backupCfg.s3.enable || backupCfg.s3.endpoint != null;
+        message = "hectic.services.project-zomboid.backup.s3.endpoint is required when S3 upload is enabled.";
+      }
+      {
+        assertion = !backupCfg.s3.enable || backupCfg.s3.region != null;
+        message = "hectic.services.project-zomboid.backup.s3.region is required when S3 upload is enabled.";
+      }
+      {
+        assertion =
+          !backupCfg.s3.enable
+          || backupCfg.s3.credentialsFile == null
+          || (
+            lib.hasPrefix "/" backupCfg.s3.credentialsFile
+            && !lib.hasPrefix "/nix/store/" backupCfg.s3.credentialsFile
+          );
+        message = "hectic.services.project-zomboid.backup.s3.credentialsFile must be a runtime path outside /nix/store.";
+      }
+      {
+        assertion =
+          !backupCfg.s3.enable
+          || backupCfg.s3.endpoint == null
+          || lib.hasPrefix "https://" backupCfg.s3.endpoint;
+        message = "hectic.services.project-zomboid.backup.s3.endpoint must use HTTPS.";
+      }
+    ];
+
+    users.groups.project-zomboid = { };
+    users.users.project-zomboid = {
+      isSystemUser = true;
+      group = "project-zomboid";
+      home = cfg.dataDir;
+      createHome = true;
+    };
+
+    systemd.tmpfiles.rules = [
+      "d ${cfg.dataDir} 0750 project-zomboid project-zomboid - -"
+      "d ${cfg.installDir} 0750 project-zomboid project-zomboid - -"
+    ] ++ lib.optionals backupCfg.enable [
+      "d ${cfg.dataDir}/backups 0700 project-zomboid project-zomboid - -"
+      "Z ${cfg.dataDir}/backups 0700 project-zomboid project-zomboid - -"
+      "d ${backupCfg.stagingDir} 0700 project-zomboid project-zomboid - -"
+      "d ${backupCfg.archiveDir} 0700 project-zomboid project-zomboid - -"
+    ];
+
+    systemd.services.project-zomboid = {
+      description = "Project Zomboid dedicated server";
+      after = [ "network-online.target" ];
+      wants = [ "network-online.target" ];
+      wantedBy = [ "multi-user.target" ];
+
+      preStart = ''
+        ${pkgs.coreutils}/bin/install -d -m 0750 \
+          ${lib.escapeShellArg cfg.installDir}
+        if [ ! -s ${lib.escapeShellArg adminPasswordFile} ]; then
+          umask 077
+          ${pkgs.openssl}/bin/openssl rand -base64 32 > ${lib.escapeShellArg adminPasswordFile}
+        fi
+        ${lib.optionalString cfg.rcon.enable ''
+          if [ ! -s ${lib.escapeShellArg rconPasswordFile} ]; then
+            umask 077
+            ${pkgs.openssl}/bin/openssl rand -hex 32 > ${lib.escapeShellArg rconPasswordFile}
+          else
+            rcon_password=$(${pkgs.coreutils}/bin/cat ${lib.escapeShellArg rconPasswordFile})
+            case "$rcon_password" in
+              *[!0123456789abcdefABCDEF]*)
+                umask 077
+                ${pkgs.openssl}/bin/openssl rand -hex 32 > ${lib.escapeShellArg rconPasswordFile}
+                ;;
+            esac
+          fi
+          ${pkgs.coreutils}/bin/chown project-zomboid:project-zomboid ${lib.escapeShellArg rconPasswordFile}
+          ${pkgs.coreutils}/bin/chmod 0600 ${lib.escapeShellArg rconPasswordFile}
+        ''}
+        ${pkgs.steamcmd}/bin/steamcmd \
+          +force_install_dir ${lib.escapeShellArg cfg.installDir} \
+          +login anonymous \
+          +app_update 380870 ${lib.optionalString (cfg.branch != null) "-beta ${lib.escapeShellArg cfg.branch}"} validate \
+          +quit
+        ${pkgs.gnused}/bin/sed -i -E \
+          's/"-Xmx[0-9]+[mMgG]"/"-Xmx${cfg.memory}"/' \
+          ${lib.escapeShellArg "${cfg.installDir}/ProjectZomboid64.json"}
+        ${pkgs.coreutils}/bin/install -d -m 0750 \
+          ${lib.escapeShellArg "${zomboidDir}/Server"}
+        {
+          ${lib.concatMapStringsSep "\n  " (line:
+            "${pkgs.coreutils}/bin/printf '%s\\n' ${lib.escapeShellArg line};"
+          ) configLines}
+          ${lib.optionalString (cfg.serverPropertiesFile != null)
+            "${pkgs.coreutils}/bin/cat ${lib.escapeShellArg cfg.serverPropertiesFile};"}
+          ${lib.optionalString cfg.rcon.enable ''
+            ${pkgs.coreutils}/bin/printf '%s\n' ${lib.escapeShellArg "RCONPort=${toString cfg.rcon.port}"};
+            ${pkgs.coreutils}/bin/printf '%s' 'RCONPassword=';
+            ${pkgs.coreutils}/bin/cat ${lib.escapeShellArg rconPasswordFile};
+            ${pkgs.coreutils}/bin/printf '\n';
+          ''}
+        } > ${lib.escapeShellArg "${zomboidDir}/Server/${cfg.serverName}.ini"}
+        ${lib.optionalString (cfg.sandboxProperties != { }) ''
+          {
+            ${pkgs.coreutils}/bin/printf '%s\n' 'SandboxVars = {';
+            ${lib.concatMapStringsSep "\n  " (line:
+              "${pkgs.coreutils}/bin/printf '%s\\n' ${lib.escapeShellArg line};"
+            ) sandboxConfigLines}
+            ${pkgs.coreutils}/bin/printf '%s\n' '};';
+          } > ${lib.escapeShellArg "${zomboidDir}/Server/${cfg.serverName}_SandboxVars.lua"}
+        ''}
+        ${lib.optionalString (cfg.sandboxProperties == { }) ''
+          ${pkgs.coreutils}/bin/rm -f \
+            ${lib.escapeShellArg "${zomboidDir}/Server/${cfg.serverName}_SandboxVars.lua"}
+        ''}
+      '';
+
+      serviceConfig = {
+        User = "project-zomboid";
+        Group = "project-zomboid";
+        WorkingDirectory = cfg.dataDir;
+        Environment = [
+          "HOME=${cfg.dataDir}"
+          "SteamAppId=108600"
+        ];
+        ExecStart = startScript;
+        Restart = "on-failure";
+        RestartSec = 5;
+        TimeoutStartSec = "15min";
+        TimeoutStopSec = 30;
+        UMask = "0077";
+      };
+    };
+
+    systemd.services.project-zomboid-backup = lib.mkIf backupCfg.enable {
+      description = "No-stop Project Zomboid backup";
+      after = [ "project-zomboid.service" ];
+      unitConfig.ConditionPathExists = [
+        saveDir
+        serverConfigDir
+      ];
+      serviceConfig = {
+        Type = "oneshot";
+        User = "project-zomboid";
+        Group = "project-zomboid";
+        ExecStart = backupScript;
+        TimeoutStartSec = "30min";
+        UMask = "0077";
+      } // lib.optionalAttrs backupCfg.s3.enable {
+        EnvironmentFile = s3CredentialsFile;
+      };
+    };
+
+    systemd.timers.project-zomboid-backup = lib.mkIf backupCfg.enable {
+      description = "Run Project Zomboid backup";
+      wantedBy = [ "timers.target" ];
+      timerConfig = {
+        OnCalendar = backupCfg.onCalendar;
+        Persistent = true;
+      };
+    };
+
+    networking.firewall.allowedUDPPorts = lib.mkIf cfg.openFirewall [
+      cfg.port
+      cfg.udpPort
+    ];
+  };
+}
@@ -147,7 +147,7 @@ in {

    (lib.mkIf cfg.watcher.enable {
      sops.secrets."sentinèlla/watcher/environment" = lib.mkDefault {
-        sopsFile = "${flake}/sus/sentinella-default.yaml";
+        sopsFile = flake + "/sus/sentinella-default.yaml";
      };

      systemd.services."sentinella-watcher" = {
@@ -0,0 +1,153 @@
+{ ... }: {
+  pkgs,
+  lib,
+  config,
+  ...
+}: let
+  cfg = config.hectic.services.stable-video-diffusion;
+in {
+  options.hectic.services.stable-video-diffusion = {
+    enable = lib.mkEnableOption "local Stable Video Diffusion HTTP API";
+
+    host = lib.mkOption {
+      type = lib.types.str;
+      default = "127.0.0.1";
+      description = "Address the Stable Video Diffusion API binds to.";
+    };
+
+    port = lib.mkOption {
+      type = lib.types.port;
+      default = 7861;
+      description = "Port the Stable Video Diffusion API binds to.";
+    };
+
+    package = lib.mkOption {
+      type = lib.types.package;
+      default = pkgs.hectic.stable-video-diffusion-api;
+      defaultText = lib.literalExpression "pkgs.hectic.stable-video-diffusion-api";
+      description = "Package providing the Stable Video Diffusion API executable.";
+    };
+
+    modelId = lib.mkOption {
+      type = lib.types.str;
+      default = "stabilityai/stable-video-diffusion-img2vid-xt";
+      description = "Model identifier loaded by the Stable Video Diffusion API.";
+    };
+
+    device = lib.mkOption {
+      type = with lib.types; nullOr (enum [ "cpu" "cuda" ]);
+      default = null;
+      description = ''
+        Torch device requested from the Stable Video Diffusion API. When null,
+        the package keeps its own auto-detection behavior.
+      '';
+    };
+
+    libraryPath = lib.mkOption {
+      type = with lib.types; listOf str;
+      default = [];
+      description = ''
+        Runtime library paths added to LD_LIBRARY_PATH. CUDA/NVIDIA deployments
+        should include /run/opengl-driver/lib so libcuda.so is visible to torch.
+      '';
+    };
+
+    stateDir = lib.mkOption {
+      type = lib.types.str;
+      default = "/var/lib/stable-video-diffusion-api";
+      description = "Persistent state directory for the Stable Video Diffusion API.";
+    };
+
+    cacheDir = lib.mkOption {
+      type = lib.types.str;
+      default = "/var/cache/stable-video-diffusion-api";
+      description = "Cache directory for downloaded model and runtime artifacts.";
+    };
+
+    outputDir = lib.mkOption {
+      type = lib.types.str;
+      default = "${cfg.stateDir}/outputs";
+      defaultText = lib.literalExpression ''"\${config.hectic.services.stable-video-diffusion.stateDir}/outputs"'';
+      description = "Directory where generated video outputs are written.";
+    };
+
+    environmentFile = lib.mkOption {
+      type = with lib.types; nullOr path;
+      default = null;
+      description = ''
+        Optional environment file for secrets or runtime overrides. Values from
+        the unit environment define SVD_API_HOST, SVD_API_PORT, SVD_MODEL_ID,
+        SVD_STATE_DIR, SVD_CACHE_DIR, SVD_OUTPUT_DIR, and optionally SVD_DEVICE
+        and LD_LIBRARY_PATH by default.
+      '';
+    };
+
+    openFirewall = lib.mkOption {
+      type = lib.types.bool;
+      default = false;
+      description = "Whether to open the API port in the firewall.";
+    };
+  };
+
+  config = lib.mkIf cfg.enable {
+    assertions = [
+      {
+        assertion = cfg.device != "cuda" || cfg.libraryPath != [];
+        message = "hectic.services.stable-video-diffusion.libraryPath must include the NVIDIA runtime library path when device is cuda.";
+      }
+    ];
+
+    users.users.stable-video-diffusion = {
+      isSystemUser = true;
+      group = "stable-video-diffusion";
+    };
+    users.groups.stable-video-diffusion = {};
+
+    systemd.tmpfiles.rules = [
+      "d ${cfg.stateDir} 0750 stable-video-diffusion stable-video-diffusion -"
+      "d ${cfg.cacheDir} 0750 stable-video-diffusion stable-video-diffusion -"
+      "d ${cfg.outputDir} 0750 stable-video-diffusion stable-video-diffusion -"
+    ];
+
+    systemd.services.stable-video-diffusion-api = {
+      description = "Stable Video Diffusion HTTP API";
+      after       = [ "network.target" ];
+      wantedBy    = [ "multi-user.target" ];
+      environment = {
+        SVD_API_HOST   = cfg.host;
+        SVD_API_PORT   = toString cfg.port;
+        SVD_MODEL_ID   = cfg.modelId;
+        SVD_STATE_DIR  = cfg.stateDir;
+        SVD_CACHE_DIR  = cfg.cacheDir;
+        SVD_OUTPUT_DIR = cfg.outputDir;
+      }
+      // lib.optionalAttrs (cfg.device != null) {
+        SVD_DEVICE = cfg.device;
+      }
+      // lib.optionalAttrs (cfg.libraryPath != []) {
+        LD_LIBRARY_PATH = lib.concatStringsSep ":" cfg.libraryPath;
+      };
+      serviceConfig = lib.mkMerge [
+        {
+          Type            = "simple";
+          User            = "stable-video-diffusion";
+          Group           = "stable-video-diffusion";
+          WorkingDirectory = cfg.stateDir;
+          ExecStart       = lib.getExe' cfg.package "stable-video-diffusion-api";
+          Restart         = "on-failure";
+          RestartSec      = "5s";
+          TimeoutStopSec  = "30s";
+          KillSignal      = "SIGTERM";
+          KillMode        = "mixed";
+          StandardOutput  = "journal";
+          StandardError   = "journal";
+        }
+        (lib.mkIf (cfg.environmentFile != null) {
+          EnvironmentFile = cfg.environmentFile;
+        })
+      ];
+    };
+
+    networking.firewall.allowedTCPPorts = lib.mkIf cfg.openFirewall [ cfg.port ];
+  };
+}
@@ -1,230 +0,0 @@
-# Matrix Cluster Failover Runbook (`accord.tube`)
-
-Primary: `hectic-lab` (NL, `128.140.75.58`)
-Standby: `bfs.poland.xray` (PL, `91.198.166.181`)
-
-Module: `hectic.generic.matrix-cluster` (`nixos/module/generic/matrix-cluster.nix`).
-Shared secrets: `sus/matrix-cluster.yaml`.
-
-All `psql` and `pg_ctl` invocations use PostgreSQL **17** at data dir
-`/var/lib/postgresql/17`.
-
-## Initial setup
-
-### 1. Provision shared SOPS file (`sus/matrix-cluster.yaml`)
-
-On a workstation with both yukkop and yukkop-alt age keys available:
-
-```sh
-sudo cat /var/lib/matrix-synapse/homeserver.signing.key  # on NL (hectic-lab)
-# Copy the single line value into the buffer for the next step.
-
-sops sus/matrix-cluster.yaml
-```
-
-Populate the editor with:
-
-```yaml
-matrix:
-  signing-key: <paste verbatim signing-key line from NL>
-  postgres-replication-password: <openssl rand -base64 32>
-  object-storage:
-    credentials: |
-      ACCESS_KEY_ID=<verbatim copy from sus/hectic-lab.yaml>
-      SECRET_ACCESS_KEY=<verbatim copy from sus/hectic-lab.yaml>
-  porkbun-api-key: <PORKBUN_API_KEY>
-  porkbun-secret-api-key: <PORKBUN_SECRET_API_KEY>
-```
-
-Verify recipients:
-
-```sh
-sops updatekeys sus/matrix-cluster.yaml
-sops -d sus/matrix-cluster.yaml | grep -E 'signing-key|porkbun-api-key|object-storage'
-```
-
-Expected: all five keys present, exit 0.
-
-### 2. Deploy NL primary first
-
-```sh
-nixos-rebuild switch --flake .#'hectic-lab|x86_64-linux' --target-host root@128.140.75.58
-```
-
-Verify on NL:
-
-```sh
-sudo systemctl status matrix-synapse postgresql matrix-cluster-replication-password
-sudo -u postgres psql -c "select rolname, rolreplication from pg_roles where rolname='replication';"
-# Expected: replication | t
-```
-
-### 3. Seed PL replica with `pg_basebackup`
-
-On PL:
-
-```sh
-sudo systemctl stop postgresql
-sudo rm -rf /var/lib/postgresql/17
-sudo -u postgres install -d -m 0700 /var/lib/postgresql/17
-sudo -u postgres PGPASSWORD="$(sudo cat /run/secrets/matrix/postgres-replication-password)" \
-  pg_basebackup \
-    -h 128.140.75.58 \
-    -p 5432 \
-    -U replication \
-    -D /var/lib/postgresql/17 \
-    -Fp -Xs -P -R \
-    --no-password
-```
-
-`-R` writes `standby.signal` and an initial `primary_conninfo`. The
-matrix-cluster module's `matrix-cluster-standby-bootstrap` service will
-overwrite `primary_conninfo` to use a libpq passfile on next boot.
-
-### 4. Deploy PL standby
-
-```sh
-nixos-rebuild switch --flake .#'bfs.poland.xray|x86_64-linux' --target-host root@91.198.166.181
-sudo systemctl start postgresql
-```
-
-Verify streaming on NL:
-
-```sh
-sudo -u postgres psql -c 'select client_addr, state, sync_state from pg_stat_replication;'
-# Expected: 91.198.166.181 | streaming | async
-```
-
-Verify standby on PL:
-
-```sh
-sudo -u postgres psql -c 'select pg_is_in_recovery();'
-# Expected: t
-sudo systemctl is-active matrix-synapse
-# Expected: inactive (standby keeps Synapse off)
-```
-
-### 5. Remove duplicate S3 credentials from `sus/hectic-lab.yaml`
-
-Only AFTER NL is confirmed healthy reading from the new shared file:
-
-```sh
-sops sus/hectic-lab.yaml
-# Delete the matrix/object-storage/credentials block.
-sudo nixos-rebuild switch --flake .#'hectic-lab|x86_64-linux'
-```
-
-## Normal operations
-
-```sh
-# NL: replication health
-sudo -u postgres psql -c 'select * from pg_stat_replication;'
-# Expected: 1 row, state=streaming, sync_state=async
-
-# PL: replay status
-sudo -u postgres psql -c 'select now() - pg_last_xact_replay_timestamp() as lag;'
-
-# Both: cert renewal
-sudo systemctl status acme-accord.tube.timer
-sudo journalctl -u acme-accord.tube.service --since '24 hours ago'
-
-# Synapse health (NL primary)
-curl -sf https://accord.tube/_matrix/client/versions | head
-```
-
-## Planned failover (NL -> PL)
-
-```sh
-# 1. Drain NL: stop accepting writes.
-sudo systemctl stop matrix-synapse
-sudo systemctl stop postgresql   # ensure no new WAL after this point
-
-# 2. Promote PL replica.
-sudo -u postgres pg_ctl -D /var/lib/postgresql/17 promote
-# Wait until pg_is_in_recovery() returns f:
-sudo -u postgres psql -c 'select pg_is_in_recovery();'
-
-# 3. Make the role switch declarative before rebuilding.
-#    Edit the flake so rebuilds match the promoted database state:
-#      - nixos/system/bfs.poland.xray/bfs.poland.xray.nix:
-#          hectic.generic.matrix-cluster.role = "primary";
-#          hectic.generic.matrix-cluster.overrideEnableSynapse = true;
-#          hectic.generic.matrix-cluster.secretsFile = config.sops.secrets."matrix/secrets".path;
-#      - nixos/system/hectic-lab/hectic-lab.nix:
-#          hectic.generic.matrix-cluster.role = "standby";
-#          hectic.generic.matrix-cluster.overrideEnableSynapse = false;
-#          hectic.generic.matrix-cluster.replication.peerHost = "91.198.166.181";
-#          hectic.generic.matrix-cluster.replication.allowedSourceIPs = [ "128.140.75.58/32" ];
-#    (You will also need a matrix/secrets entry on PL - copy from NL via SOPS.)
-sudo nixos-rebuild switch --flake .#'bfs.poland.xray|x86_64-linux'
-sudo nixos-rebuild switch --flake .#'hectic-lab|x86_64-linux'
-sudo systemctl status matrix-synapse
-
-# 4. Swap DNS A record at Porkbun:
-#    accord.tube  A  91.198.166.181   (was 128.140.75.58)
-#    TTL: set to 300 in advance of any planned failover.
-#    Porkbun UI: https://porkbun.com/account/domainsSpeedy -> accord.tube -> DNS -> edit A record.
-#    Or via API:
-sudo curl -sX POST https://api.porkbun.com/api/json/v3/dns/editByNameType/accord.tube/A \
-  -H 'content-type: application/json' \
-  -d "$(jq -n --arg k "$PORKBUN_API_KEY" --arg s "$PORKBUN_SECRET_API_KEY" \
-        '{secretapikey:$s,apikey:$k,content:"91.198.166.181",ttl:"300"}')"
-
-# 5. Federation smoke test.
-curl -s 'https://federationtester.matrix.org/api/report?server_name=accord.tube' | jq .FederationOK
-# Expected: true
-```
-
-Expected after the rebuilds:
-
- `bfs.poland.xray` evaluates and runs as `role = "primary"`.
- `hectic-lab` evaluates as `role = "standby"` with Synapse forced off.
- Future `nixos-rebuild` runs preserve the promoted topology instead of reapplying standby settings to PL.
-
-## Failback (PL -> NL)
-
-```sh
-# 1. Stop NL postgres if still up; clear its data dir.
-sudo systemctl stop postgresql matrix-synapse
-sudo rm -rf /var/lib/postgresql/17
-
-# 2. Re-seed NL from PL (now the live primary).
-sudo -u postgres install -d -m 0700 /var/lib/postgresql/17
-sudo -u postgres PGPASSWORD="$(sudo cat /run/secrets/matrix/postgres-replication-password)" \
-  pg_basebackup -h 91.198.166.181 -p 5432 -U replication \
-    -D /var/lib/postgresql/17 -Fp -Xs -P -R --no-password
-
-# 3. Temporarily flip roles in the flake:
-#    - hectic-lab.nix: role = "standby";   peerHost = "91.198.166.181";
-#    - bfs.poland.xray.nix: role = "primary"; peerHost = "128.140.75.58";
-#    Rebuild both.
-
-# 4. Once NL is streaming green, do the reverse failover dance:
-sudo systemctl stop matrix-synapse                    # on PL
-sudo -u postgres pg_ctl -D /var/lib/postgresql/17 promote   # on NL
-# Then revert the flake role assignments back to NL=primary / PL=standby and
-# rebuild both hosts.
-
-# 5. Swap DNS back at Porkbun (A -> 128.140.75.58).
-```
-
-## Disaster recovery (NL permanently lost)
-
-```sh
-# 1. Promote PL as the new permanent primary.
-sudo -u postgres pg_ctl -D /var/lib/postgresql/17 promote
-
-# 2. Edit nixos/system/bfs.poland.xray/bfs.poland.xray.nix:
-#      hectic.generic.matrix-cluster.role = "primary";
-#      hectic.generic.matrix-cluster.overrideEnableSynapse = lib.mkForce null;
-#      hectic.generic.matrix-cluster.replication.peerHost = "<new-standby-ip>";
-#      hectic.generic.matrix-cluster.replication.allowedSourceIPs = [ "<new-standby-ip>/32" ];
-
-# 3. Provision a new host (any region with Porkbun-managed DNS) and import
-#    self.nixosModules.matrix-cluster with role = "standby" pointed at PL's IP.
-
-# 4. Bootstrap the new standby via pg_basebackup from PL exactly as in
-#    "Initial setup" step 3, replacing 128.140.75.58 with PL's IP.
-
-# 5. Update Porkbun A record to PL's IP permanently.
-```
@@ -9,18 +9,14 @@
  config,
  ...
 }: let
-  matrixBackend = "https://128.140.75.58";
  matrixHost = "accord.tube";
  jitsiHost = "meet.accord.tube";
  elementEntryDomain = "element.bfs.band";
  polandEntryDomain = "bfs.band";
-  backendProxyConfig = ''
-    proxy_ssl_server_name on;
-    proxy_ssl_name ${matrixHost};
-    proxy_set_header Host ${matrixHost};
-  '';
+  matrixClusterSopsFile = flake + "/sus/matrix-cluster.yaml";
 in {
  imports = [
+    ./minecraft-wow.nix
    self.nixosModules.xray-system
    self.nixosModules.matrix-cluster
    self.nixosModules.matrix-cluster-users
@@ -46,7 +42,6 @@ in {
      credentialsFile = config.sops.secrets."matrix/object-storage/credentials".path;
    };
    replication = {
-      peerHost     = "128.140.75.58";
      passwordFile = config.sops.secrets."matrix/postgres-replication-password".path;
    };
    acme = {
@@ -71,11 +66,31 @@ in {
    hostName = jitsiHost;
  };

+  # NOTE(yukkop): disk was provisioned outside disko, so the expected partition
+  # label does not exist. Pin root to the live filesystem UUID so stage 1 can
+  # mount `/` reliably.
+  fileSystems."/" = lib.mkForce {
+    device = "/dev/disk/by-uuid/06b48ef1-a1eb-428d-821c-90c96a624542";
+    fsType = "ext4";
+  };
+
  security.acme = {
    acceptTerms = true;
    defaults.email = "security@bfs.band";
  };

+  # NOTE(yukkop): this host gets an IPv6 route via RA, but object storage
+  # fetches to hel1.your-objectstorage.com currently stall over IPv6 while
+  # IPv4 works. Synapse's S3 media backend uses getaddrinfo ordering, so
+  # prefer IPv4 here to keep Element media downloads responsive.
+  environment.etc."gai.conf".text = ''
+    precedence ::ffff:0:0/96  100
+  '';
+
+  systemd.services.matrix-synapse.restartTriggers = [
+    config.environment.etc."gai.conf".source
+  ];
+
  services.nginx = {
    enable = true;

@@ -112,19 +127,17 @@ in {
      };

      locations."= /livekit/jwt" = {
-        proxyPass = "${matrixBackend}/livekit/jwt";
-        extraConfig = backendProxyConfig;
+        proxyPass = "http://[::1]:${toString config.services.lk-jwt-service.port}/";
      };

      locations."^~ /livekit/jwt/" = {
-        proxyPass = "${matrixBackend}/livekit/jwt/";
-        extraConfig = backendProxyConfig;
+        proxyPass = "http://[::1]:${toString config.services.lk-jwt-service.port}/";
      };

      locations."= /livekit/sfu" = {
-        proxyPass = "${matrixBackend}/livekit/sfu";
+        proxyPass = "http://[::1]:${toString config.services.livekit.settings.port}/";
        proxyWebsockets = true;
-        extraConfig = backendProxyConfig + ''
+        extraConfig = ''
          proxy_send_timeout 120;
          proxy_read_timeout 120;
          proxy_buffering off;
@@ -135,9 +148,9 @@ in {
      };

      locations."^~ /livekit/sfu/" = {
-        proxyPass = "${matrixBackend}/livekit/sfu/";
+        proxyPass = "http://[::1]:${toString config.services.livekit.settings.port}/";
        proxyWebsockets = true;
-        extraConfig = backendProxyConfig + ''
+        extraConfig = ''
          proxy_send_timeout 120;
          proxy_read_timeout 120;
          proxy_buffering off;
@@ -148,13 +161,14 @@ in {
      };

      locations."^~ /_matrix/" = {
-        proxyPass = "${matrixBackend}/_matrix/";
-        extraConfig = backendProxyConfig;
+        proxyPass = "http://127.0.0.1:8008";
+        extraConfig = ''
+          client_max_body_size ${config.hectic.generic.matrix-cluster.maxUploadSize};
+        '';
      };

      locations."^~ /_synapse/client/" = {
-        proxyPass = "${matrixBackend}/_synapse/client/";
-        extraConfig = backendProxyConfig;
+        proxyPass = "http://127.0.0.1:8008";
      };
    };

@@ -164,7 +178,7 @@ in {

      locations."= /config.${elementEntryDomain}.json".return = "302 /config.json";

-      root = pkgs.element-web.override {
+      root = pkgs.hectic.element-web.override {
        conf = {
          default_server_config = {
            "m.homeserver".base_url = "https://${polandEntryDomain}";
@@ -176,6 +190,8 @@ in {
            preferred_domain = jitsiHost;
          };

+          hectic.videoMessages.enabled = true;
+
          room_directory.servers = [ matrixHost ];

          default_theme = "dark";
@@ -189,41 +205,41 @@ in {
    key      = "matrix/signing-key";
    owner    = "matrix-synapse";
    mode     = "0400";
-    sopsFile = "${flake}/sus/matrix-cluster.yaml";
+    sopsFile = matrixClusterSopsFile;
  };
  sops.secrets."matrix/postgres-replication-password" = {
    key      = "matrix/postgres-replication-password";
    owner    = "postgres";
    mode     = "0400";
-    sopsFile = "${flake}/sus/matrix-cluster.yaml";
+    sopsFile = matrixClusterSopsFile;
  };
  sops.secrets."matrix/object-storage/credentials" = {
    key      = "matrix/object-storage/credentials";
    owner    = "matrix-synapse";
    mode     = "0400";
-    sopsFile = "${flake}/sus/matrix-cluster.yaml";
+    sopsFile = matrixClusterSopsFile;
  };
  sops.secrets."matrix/secrets" = {
    key      = "matrix/secrets";
    owner    = "matrix-synapse";
    mode     = "0400";
-    sopsFile = "${flake}/sus/matrix-cluster.yaml";
+    sopsFile = matrixClusterSopsFile;
  };
  sops.secrets."matrix/turn-secret" = {
    key      = "matrix/turn-secret";
    owner    = "turnserver";
    group    = "turnserver";
    mode     = "0400";
-    sopsFile = "${flake}/sus/matrix-cluster.yaml";
+    sopsFile = matrixClusterSopsFile;
  };
  sops.secrets."matrix/porkbun-api-key" = {
    key      = "matrix/porkbun-api-key";
    mode     = "0400";
-    sopsFile = "${flake}/sus/matrix-cluster.yaml";
+    sopsFile = matrixClusterSopsFile;
  };
  sops.secrets."matrix/porkbun-secret-api-key" = {
    key      = "matrix/porkbun-secret-api-key";
    mode     = "0400";
-    sopsFile = "${flake}/sus/matrix-cluster.yaml";
+    sopsFile = matrixClusterSopsFile;
  };
 }
@@ -0,0 +1,35 @@
+{ ... }:
+{
+  imports = [ (import ../../module/generic/minecraft-public-relay.nix { }) ];
+  services.minecraft-public-relay = {
+    enable = true;
+    publicKey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIKNWWegOVTOF3EOmam32iP7sMybULMTxsXuC+cEGITQ8 minecraft-wow-relay";
+  };
+  systemd.tmpfiles.rules = [ "d /var/www/store/minecraft/world-of-sosal 0755 root root -" ];
+  services.nginx.virtualHosts."store.bfs.band" = {
+    enableACME = true;
+    forceSSL = true;
+    root = "/var/www/store";
+    locations."= /".return = "302 /minecraft/world-of-sosal/";
+    locations."= /minecraft".return = "302 /minecraft/world-of-sosal/";
+    locations."= /minecraft/".return = "302 /minecraft/world-of-sosal/";
+    locations."/".extraConfig = ''
+      autoindex off;
+      add_header Cache-Control "no-cache";
+      try_files $uri $uri/ =404;
+    '';
+  };
+  # Keep old pack URLs working for already imported Prism instances.
+  services.nginx.virtualHosts."bfs.band".locations = {
+    "= /minecraft".return = "302 /minecraft/world-of-sosal/";
+    "= /minecraft/".return = "302 /minecraft/world-of-sosal/";
+    "^~ /minecraft/" = {
+      root = "/var/www/store";
+      extraConfig = ''
+        autoindex off;
+        add_header Cache-Control "no-cache";
+        try_files $uri $uri/ =404;
+      '';
+    };
+  };
+}
@@ -38,6 +38,13 @@

  environment.systemPackages = (with pkgs; [ rsync git steamcmd hectic.AstroTuxLauncher ]);

+  hectic.services."project-zomboid" = {
+    enable = true;
+    serverName = "servertest";
+    workshopItems = [ ];
+    mods = [ ];
+  };
+
  users.users.root.openssh.authorizedKeys.keys = [
    ''ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIKAaObjLBslsdTlqEcYaS1TqX4x9aVJu75y27/8MFevO''
  ];
@@ -0,0 +1,156 @@
+{
+  domain,
+  ...
+}: {
+  config,
+  lib,
+  pkgs,
+  ...
+}: let
+  repackedActive = true;
+  migrationWriteFreeze = false;
+
+  uploadProxyConfig = ''
+    # Stream large NARs and tolerate S3 backpressure while Attic reads them.
+    proxy_http_version 1.1;
+    proxy_request_buffering off;
+    proxy_send_timeout 600s;
+    proxy_read_timeout 600s;
+  '';
+
+  repackedSettings = config.services.atticd.settings // {
+    listen = "127.0.0.1:8082";
+    allowed-hosts = [ "cache.${domain}" ];
+    api-endpoint = if repackedActive then "https://cache.${domain}/" else "https://cache.${domain}/next/";
+    substituter-endpoint = if repackedActive then "https://cache.${domain}/" else "https://cache.${domain}/next/";
+    database.url = "sqlite:///var/lib/atticd-repacked/server.db?mode=rwc";
+    storage = {
+      type     = "s3";
+      bucket   = "nix-cache-hectic-lab";
+      endpoint = "https://hel1.your-objectstorage.com";
+      region   = "hel1";
+    };
+    chunking = {
+      nar-size-threshold = 1048576;
+      min-size           = 1048576;
+      avg-size           = 2097152;
+      max-size           = 4194304;
+    };
+    compression.type = "zstd";
+  };
+
+  repackedConfigFile = pkgs.runCommand "checked-atticd-repacked.toml" {
+    configFile = (pkgs.formats.toml { }).generate "server-repacked.toml" repackedSettings;
+  } ''
+    export ATTIC_SERVER_TOKEN_RS256_SECRET_BASE64="$(${lib.getExe pkgs.openssl} genrsa -traditional 4096 | ${pkgs.coreutils}/bin/base64 -w0)"
+    export ATTIC_SERVER_DATABASE_URL="sqlite://:memory:"
+    ${lib.getExe config.services.atticd.package} --mode check-config -f $configFile
+    cat <$configFile >$out
+  '';
+in {
+  hectic.services.attic = {
+    enable          = true;
+    hostName        = "cache.${domain}";
+    port            = 8081;
+    environmentFile = config.sops.secrets."atticd/environment".path;
+    storage = {
+      bucket   = "cache-hectic-lab";
+      endpoint = "https://hel1.your-objectstorage.com";
+      region   = "hel1";
+    };
+  };
+
+  # Slow S3 chunk reads can exceed the SDK's default 20-second stall grace.
+  services.atticd.package = pkgs.attic-server.overrideAttrs (old: {
+    # Restrict the SDK TLS connector to HTTP/1.1 after S3 REFUSED_STREAM errors.
+    cargoDeps = pkgs.runCommand "attic-cargo-vendor-http1" { } ''
+      mkdir "$out"
+      shopt -s dotglob
+      for entry in ${old.cargoDeps}/*; do
+        ln -s "$entry" "$out/$(basename "$entry")"
+      done
+      crate=aws-smithy-http-client-1.0.6
+      rm "$out/$crate"
+      cp -rL ${old.cargoDeps}/"$crate" "$out/$crate"
+      chmod -R u+w "$out/$crate"
+      substituteInPlace "$out/$crate/src/client/tls.rs" \
+        --replace-fail '.enable_http2()' ""
+    '';
+    postPatch = (old.postPatch or "") + ''
+      substituteInPlace server/src/storage/s3.rs \
+        --replace-fail 'let mut builder = S3ConfigBuilder::from(&shared_config);' \
+          'let mut builder = S3ConfigBuilder::from(&shared_config)
+              .stalled_stream_protection(
+                  aws_sdk_s3::config::StalledStreamProtectionConfig::enabled()
+                      .grace_period(Duration::from_secs(120))
+                      .build(),
+              );'
+    '';
+  });
+
+  services.atticd.settings = lib.mkIf repackedActive {
+    api-endpoint = lib.mkForce "https://cache.${domain}/previous/";
+    substituter-endpoint = "https://cache.${domain}/previous/";
+  };
+  services.atticd.mode = if migrationWriteFreeze || repackedActive then "api-server" else "monolithic";
+
+  systemd.services.atticd-repacked = {
+    wantedBy = [ "multi-user.target" ];
+    after    = [ "network-online.target" ];
+    wants    = [ "network-online.target" ];
+
+    serviceConfig = config.systemd.services.atticd.serviceConfig // {
+      ExecStart       = "${lib.getExe config.services.atticd.package} -f ${repackedConfigFile} --mode monolithic";
+      EnvironmentFile = config.sops.secrets."atticd/environment".path;
+      StateDirectory  = "atticd-repacked";
+      User            = "atticd-repacked";
+      Group           = "atticd-repacked";
+    };
+  };
+
+  services.nginx.virtualHosts."cache.${domain}" = {
+    enableACME = true;
+    forceSSL   = true;
+    extraConfig = ''
+      client_max_body_size 0;
+    '';
+    locations."/" = {
+      proxyPass = if repackedActive then "http://127.0.0.1:8082" else "http://127.0.0.1:8081";
+      extraConfig = ''
+        # Allow quiet periods while Attic fetches NAR chunks from object storage.
+        proxy_read_timeout 300s;
+      '' + lib.optionalString (migrationWriteFreeze && !repackedActive) ''
+        # Quiesce the old writer during the final snapshot and verification.
+        limit_except GET {
+          deny all;
+        }
+      '';
+    };
+    locations."/next/" = {
+      proxyPass = "http://127.0.0.1:8082/";
+      extraConfig = ''
+        # Allow quiet periods while Attic fetches NAR chunks from object storage.
+        proxy_read_timeout 300s;
+      '';
+    };
+    locations."= /_api/v1/upload-path" = lib.mkIf (repackedActive || !migrationWriteFreeze) {
+      proxyPass = if repackedActive then "http://127.0.0.1:8082" else "http://127.0.0.1:8081";
+      extraConfig = uploadProxyConfig;
+    };
+    locations."= /next/_api/v1/upload-path" = {
+      proxyPass = "http://127.0.0.1:8082/_api/v1/upload-path";
+      extraConfig = uploadProxyConfig;
+    };
+    locations."/previous/" = {
+      proxyPass = "http://127.0.0.1:8081/";
+      extraConfig = ''
+        # Legacy backend is exposed for read-only migration checks.
+        limit_except GET {
+          deny all;
+        }
+        # Allow quiet periods while Attic fetches NAR chunks from object storage.
+        proxy_read_timeout 300s;
+      '';
+    };
+  };
+}
@@ -12,6 +12,8 @@ in self.lib.nixpkgs-lib.nixosSystem {
  pkgs = import inputs.nixpkgs {
    inherit system;
    overlays = [ self.overlays.default ];
+    config.allowUnfreePredicate = pkg:
+      builtins.elem (inputs.nixpkgs.lib.getName pkg) [ "steamcmd" "steam-unwrapped" "p4d" "p4" ];
  };
  modules = [
    { networking.hostName = name; }
@@ -0,0 +1,42 @@
+{
+  domain,
+  ...
+}: {
+  config,
+  ...
+}: let
+  enteDomain = "ente.${domain}";
+in {
+  hectic.services.ente = {
+    enable              = true;
+    apiDomain           = "api.${enteDomain}";
+    disableRegistration = false;
+
+    domains = {
+      accounts = "accounts.${enteDomain}";
+      cast     = "cast.${enteDomain}";
+      albums   = "albums.${enteDomain}";
+      photos   = "photos.${enteDomain}";
+    };
+
+    smtp = {
+      enable = true;
+      host   = "mail.${domain}";
+      email  = "security@${domain}";
+    };
+
+    storage = {
+      bucket   = "ente-hectic-lab";
+      endpoint = "https://hel1.your-objectstorage.com";
+      region   = "hel1";
+    };
+
+    secrets = {
+      encryptionKeyFile = config.sops.secrets."ente/key-encryption".path;
+      hashKeyFile       = config.sops.secrets."ente/key-hash".path;
+      jwtSecretFile     = config.sops.secrets."ente/jwt-secret".path;
+      s3AccessKeyFile   = config.sops.secrets."ente/s3-access-key".path;
+      s3SecretKeyFile   = config.sops.secrets."ente/s3-secret-key".path;
+    };
+  };
+}
@@ -0,0 +1,74 @@
+{ pkgs, ... }:
+let
+  port = 22222;
+  stateDir = "/var/lib/experimental-sshd";
+  configFile = pkgs.writeText "experimental-sshd_config" ''
+    Port 22222
+    ListenAddress 0.0.0.0
+    ListenAddress ::
+
+    HostKey ${stateDir}/ssh_host_ed25519_key
+    HostKey ${stateDir}/ssh_host_rsa_key
+
+    PasswordAuthentication no
+    KbdInteractiveAuthentication no
+    PubkeyAuthentication yes
+    PermitRootLogin prohibit-password
+    UsePAM yes
+    AuthenticationMethods publickey
+    AuthorizedKeysFile %h/.ssh/authorized_keys /etc/ssh/authorized_keys.d/%u
+    LogLevel DEBUG3
+
+    VersionAddendum none
+    HostKeyAlgorithms rsa-sha2-512,rsa-sha2-256,ssh-ed25519
+    KexAlgorithms curve25519-sha256,diffie-hellman-group14-sha256
+    Ciphers aes256-ctr,aes128-ctr
+    MACs hmac-sha2-512,hmac-sha2-256
+  '';
+
+  keygenScript = pkgs.writeShellScript "experimental-sshd-keygen" ''
+    set -eu
+
+    ${pkgs.coreutils}/bin/mkdir -p -m 0700 ${stateDir}
+
+    if [ ! -f ${stateDir}/ssh_host_ed25519_key ]; then
+      ${pkgs.openssh}/bin/ssh-keygen -t ed25519 -f ${stateDir}/ssh_host_ed25519_key -N ""
+    fi
+
+    if [ ! -f ${stateDir}/ssh_host_rsa_key ]; then
+      ${pkgs.openssh}/bin/ssh-keygen -t rsa -b 4096 -f ${stateDir}/ssh_host_rsa_key -N ""
+    fi
+  '';
+in
+{
+  environment.etc."ssh/experimental-sshd_config".source = configFile;
+
+  networking.firewall.allowedTCPPorts = [ port ];
+
+  systemd.services.experimental-sshd-keygen = {
+    description = "Generate experimental SSH host keys";
+    wantedBy = [ "multi-user.target" ];
+    serviceConfig = {
+      Type = "oneshot";
+      RemainAfterExit = true;
+      StateDirectory = "experimental-sshd";
+      ExecStart = keygenScript;
+    };
+  };
+
+  systemd.services.experimental-sshd = {
+    description = "Experimental SSH daemon";
+    wantedBy = [ "multi-user.target" ];
+    wants = [ "experimental-sshd-keygen.service" ];
+    after = [ "network.target" "experimental-sshd-keygen.service" ];
+    serviceConfig = {
+      Type = "simple";
+      StateDirectory = "experimental-sshd";
+      RuntimeDirectory = "experimental-sshd";
+      ExecStart = "${pkgs.openssh}/bin/sshd -D -e -f /etc/ssh/experimental-sshd_config";
+    };
+    preStart = ''
+      ${pkgs.openssh}/bin/sshd -t -f /etc/ssh/experimental-sshd_config
+    '';
+  };
+}
@@ -8,14 +8,13 @@
  config,
  pkgs,
  lib,
-  modulesPath,
  ...
 }:
 with builtins;
 with lib;
 let
  domain = "hectic-lab.com";
-  matrixDomain = "accord.tube";
+  sshPort = 22;
  mailUserNames = [
    "security"
    "founders"
@@ -25,6 +24,7 @@ let
    "iana-perlyk"
    "snuff"
    "antoshka"
+    "evgenii-kazakov"
  ];
  mkMailPasswordSecret = name: {
    name  = "mailserver/${name}/hashedPassword";
@@ -36,10 +36,19 @@ let
      hashedPasswordFile = config.sops.secrets."mailserver/${name}/hashedPassword".path;
    };
  };
-  sslOpts = {
-    sslCertificate    = config.sops.secrets."ssl/porkbun/${domain}/domain.cert.pem".path;
-    sslCertificateKey = config.sops.secrets."ssl/porkbun/${domain}/private.key.pem".path;
+  mkEnteSecret = name: {
+    name  = "ente/${name}";
+    value = {
+      owner = "ente";
+      group = "ente";
+    };
  };
+  giteaRunnerInstance = "hectic-lab-local";
+  giteaRunnerEscapedInstance = builtins.replaceStrings [ "-" ] [ "\\x2d" ] giteaRunnerInstance;
+  giteaRunnerService = "gitea-runner-${giteaRunnerEscapedInstance}";
+  giteaRunnerTokenEnvService = "${giteaRunnerService}-token-env";
+  giteaRunnerTokenEnv = "/run/gitea-runner-${giteaRunnerInstance}/token.env";
+  worldOfSosalRoot = "/var/www/store/world-of-sosal";
 in {
  imports = [
    self.nixosModules.hectic
@@ -50,10 +59,16 @@ in {
    self.nixosModules."shadowsocks"      # NOTE(nrv): usage/instance

    inputs.hectic-landing.nixosModules.hectic-landing
+    inputs.iana-angl.nixosModules.iana-angl

+    (import ./attic.nix              { inherit flake self inputs domain; })
    (import ./containers.nix          { inherit flake self inputs; })
-    (import ./mechabellum.nix         { inherit flake self inputs domain sslOpts; })
-    (import (./. + "/sentinèlla.nix") { inherit flake self inputs domain sslOpts; })
+    ./experimental-sshd.nix
+    ./minecraft-wow-proxy.nix
+    (import ./ente.nix               { inherit domain; })
+    (import ./immich.nix             { inherit domain; })
+    (import ./mechabellum.nix         { inherit flake self inputs domain; })
+    (import (./. + "/sentinèlla.nix") { inherit flake self inputs domain; })
  ];

  services.hectic-landing = {
@@ -64,6 +79,12 @@ in {
    host    = "127.0.0.1";
  };

+  services.iana-angl = {
+    enable  = true;
+    package = inputs.iana-angl.packages.${pkgs.stdenv.hostPlatform.system}.iana-angl;
+    domain  = "lessons.${domain}";
+  };
+
  # NOTE(yukkop): both nixos-mailserver and hectic-landing module set
  # security.acme.defaults.email. Force the mailserver-aligned address.
  security.acme.defaults.email = lib.mkForce "security@${domain}";
@@ -74,11 +95,122 @@ in {
      enable                 = true;
      networkMatchConfigName = "enp1s0";
      ipv4                   = "128.140.75.58";
+      floatingIpv4           = "78.47.243.0";
      ipv6                   = "2a01:4f8:c2c:d54a";
    };
    services.matrix = {
      enable = false;
    };
+    services."project-zomboid" = {
+      enable = true;
+      memory = "4g";
+      serverName = "servertest";
+      serverPropertiesFile = /var/lib/project-zomboid/server-password.ini;
+      rcon.enable = true;
+      backup = {
+        enable = true;
+        onCalendar = "*:0/30";
+        retentionDays = 14;
+        s3 = {
+          enable = true;
+          bucket = "backup-hectic-lab";
+          endpoint = "https://hel1.your-objectstorage.com";
+          region = "hel1";
+          credentialsFile = "/var/lib/project-zomboid/s3-credentials";
+        };
+      };
+      serverProperties = {
+        Map = "Muldraugh, KY";
+        SaveWorldEveryMinutes = 15;
+        DoLuaChecksum = false;
+        Public = true;
+        AntiCheatSafety = 4;
+        AntiCheatMovement = 4;
+        AntiCheatSpeed = 4;
+        AntiCheatHit = 4;
+        AntiCheatPacket = 4;
+        AntiCheatPacketException = 4;
+        AntiCheatPermission = 4;
+        AntiCheatXP = 4;
+        AntiCheatFire = 4;
+        AntiCheatSafeHouse = 4;
+        AntiCheatRecipe = 4;
+        AntiCheatPlayer = 4;
+        AntiCheatChecksum = 4;
+        AntiCheatItem = 4;
+        AntiCheatNoClip = 4;
+        AntiCheatServerCustomization = 4;
+      };
+      workshopItems = [
+        "3676456221" # Lua Digital Watch Framework
+        "3600401184" # Realistic Temperature Mod
+      ];
+      mods = [
+        "\\LuaDigitalWatchUI"
+        "\\RC_RealisticColdMod"
+      ];
+    sandboxProperties = {
+      StartMonth = 12;
+      StartDay = 1;
+      WaterShut = 3;
+      WaterShutModifier = 150;
+      ElecShut = 3;
+      ElecShutModifier = 150;
+      MinutesPerPage = 0.5;
+      Zombies = 4;
+      ZombieConfig = {
+        PopulationMultiplier = 1.3;
+        PopulationStartMultiplier = 1.0;
+        PopulationPeakMultiplier = 1.0;
+        RespawnHours = 0.0;
+        RespawnUnseenHours = 0.0;
+        RespawnMultiplier = 0.0;
+        RedistributeHours = 0.0;
+      };
+      ZombieLore = {
+        Transmission = 4;
+        Mortality = 7;
+        Speed = 2;
+        SprinterPercentage = 0;
+        Strength = 2;
+        Cognition = 2;
+        DoorOpeningPercentage = 10;
+      };
+    };
+  };
+    services.p4d = {
+      enable = true;
+      package = pkgs.p4d;
+      clientPackage = pkgs.p4;
+      openFirewall = true;
+      bootstrap.enable = false;
+    };
+    services.gitea-runner-controller = {
+      # NOTE(yukkop): ephemeral Hetzner VM runners (1 VM = 1 job).
+      # Runbook: infra/gitea-runners/runbook.md "Ephemeral VM runner cutover".
+      enable  = true;
+      budgetEurMonthly = "30";
+      imageId = "429747473"; # MicroOS x86 + persistent controller SSH key and writable Nix mount
+      armImageId = "423979717"; # OpenSUSE MicroOS ARM K3S 2026-08-24 snapshot
+      nixImageId = "161547269"; # Ubuntu 24.04 x86; Nix needs writable root
+      armNixImageId = "161547270"; # Ubuntu 24.04 ARM; Nix needs writable root
+      allowedRepos = [
+        "hinterland/*"
+        "yukkop/*"
+        "hectic-lab/*"
+      ];
+      # FIXME(yukkop): debug key for bootstrap debugging; remove once E2E stable.
+    debugSshPublicKey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIBSWIv80pyCMDQ6zH34P2qWizpOcO7X86BVhMGtbob9U gcr-controller@hectic-lab";
+    hcloudSshKeyId = 118512401;
+    };
+  };
+
+  zramSwap = {
+    enable        = true;
+    priority      = 100;
+    algorithm     = lib.mkDefault "zstd";
+    swapDevices   = 1;
+    memoryPercent = lib.mkDefault 100;
  };

  # NOTE(yukkop): disk was provisioned by Hetzner rescue image, disko was never
@@ -94,12 +226,19 @@ in {
    options = [ "umask=0077" ];
  };

+  fileSystems."/nix" = lib.mkForce {
+    device = "/dev/disk/by-id/scsi-0HC_Volume_106777875";
+    fsType = "ext4";
+    neededForBoot = true;
+  };
+
  programs.zsh.enable = true;
  programs.zsh.interactiveShellInit = ''
    setopt vi
  '';

  environment.systemPackages = with pkgs; [
+    tcpdump
    git
    rsync
    python311
@@ -110,28 +249,71 @@ in {
  sops = {
    gnupg.sshKeyPaths  = [ ];
    age.sshKeyPaths    = [ "/etc/ssh/ssh_host_ed25519_key" ];
-    defaultSopsFile    = "${flake}/sus/hectic-lab.yaml";
+    defaultSopsFile    = flake + "/sus/hectic-lab.yaml";
    secrets = builtins.listToAttrs (map mkMailPasswordSecret mailUserNames) // {
      "init-postgresql" = {
        key = "init-postgresql";
      };
-      "ssl/porkbun/${domain}/domain.cert.pem" = {
-        group = "nginx";
-        mode  = "0440";
+      "atticd/environment" = {};
+       "immich/storage-box" = {};
+       "wg-bfs/private-key" = {};
+      "gitea-runner/org-registration-token" = {
+        sopsFile = flake + "/sus/gitea-runners.yaml";
+        key      = "gitea/hectic-lab/org-runner-registration-token";
      };
-      "ssl/porkbun/${domain}/private.key.pem" = {
-        group = "nginx";
-        mode  = "0440";
+    } // builtins.listToAttrs (map mkEnteSecret [
+      "key-encryption"
+      "key-hash"
+      "jwt-secret"
+      "s3-access-key"
+      "s3-secret-key"
+    ]) // {
+      "project-zomboid/s3-access-key" = {
+        key = "ente/s3-access-key";
+        owner = "project-zomboid";
+        group = "project-zomboid";
      };
-      "ssl/porkbun/${domain}/public.key.pem" = {
-        group = "nginx";
-        mode  = "0440";
+      "project-zomboid/s3-secret-key" = {
+        key = "ente/s3-secret-key";
+        owner = "project-zomboid";
+        group = "project-zomboid";
      };
-      "wg-bfs/private-key" = {};
    };
  };

+  systemd.services.project-zomboid.preStart = lib.mkBefore ''
+    password_file=${lib.escapeShellArg "/var/lib/project-zomboid/server-password"}
+    properties_file=${lib.escapeShellArg "/var/lib/project-zomboid/server-password.ini"}
+    s3_credentials_file=${lib.escapeShellArg "/var/lib/project-zomboid/s3-credentials"}
+    s3_credentials_tmp="$(${pkgs.coreutils}/bin/mktemp "''${s3_credentials_file}.XXXXXX")"
+    trap '${pkgs.coreutils}/bin/rm -f "$s3_credentials_tmp"' EXIT
+
+    {
+      ${pkgs.coreutils}/bin/printf 'AWS_ACCESS_KEY_ID='
+      ${pkgs.coreutils}/bin/cat ${lib.escapeShellArg config.sops.secrets."project-zomboid/s3-access-key".path}
+      ${pkgs.coreutils}/bin/printf '\n'
+      ${pkgs.coreutils}/bin/printf 'AWS_SECRET_ACCESS_KEY='
+      ${pkgs.coreutils}/bin/cat ${lib.escapeShellArg config.sops.secrets."project-zomboid/s3-secret-key".path}
+      ${pkgs.coreutils}/bin/printf '\n'
+    } > "$s3_credentials_tmp"
+    ${pkgs.coreutils}/bin/chmod 0400 "$s3_credentials_tmp"
+    ${pkgs.coreutils}/bin/mv -f "$s3_credentials_tmp" "$s3_credentials_file"
+
+    if [ ! -s "$password_file" ] || ! ${pkgs.gnugrep}/bin/grep -Eq '^[0-9a-f]{48}$' "$password_file"; then
+      umask 077
+      ${pkgs.openssl}/bin/openssl rand -hex 24 > "$password_file"
+    fi
+    ${pkgs.coreutils}/bin/chmod 0600 "$password_file"
+
+    properties_file_tmp="$(${pkgs.coreutils}/bin/mktemp "$(dirname "$properties_file")/.server-password.ini.XXXXXX")"
+    ${pkgs.coreutils}/bin/printf 'Password=%s\n' "$(<"$password_file")" > "$properties_file_tmp"
+    ${pkgs.coreutils}/bin/chmod 0600 "$properties_file_tmp"
+    ${pkgs.coreutils}/bin/mv "$properties_file_tmp" "$properties_file"
+  '';
+
  users.users.root.openssh.authorizedKeys.keys = [
+    # neuro machine
+    "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIDfqSROY+rp7amPPiArY3sZM7jTjYBS02csWxF/NeIr/ root@neuro"
    # yukkop
    "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIMuP5NSfEQmO6m77xBWZvZ3hk7cw1q2k2vbsFd37rybU u0_a327@localhost"
    "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIJBLxMo5icX2Xyng7mcWGnIi+c4ZbVygjPhuU8noCkfZ"
@@ -142,6 +324,8 @@ in {
    "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIE/EhBI6sJb2yHbTkqhZiCzUrsLE6t+CZe7RhS22z7w5 nrv@adamantia"
    # github workflow
    "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIKPEUArBxu7NUULT7Pi8ArtVxY1uVbIBSaeRKtqz1sz1"
+    # gitea workflow
+    "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIAogEr5boewtUrOeOqI96y/7FWR03vdbGW93Nj01tiIS gitea-actions-hectic-lab-deploy"
  ];

  users.users.ds4d = { # NOTE(nrv): artishoque
@@ -158,6 +342,8 @@ in {
    ];
  };

+  services.openssh.ports = [ sshPort ];
+
  services.mailserver = {
    enable = true;
    domain = domain;
@@ -178,10 +364,10 @@ in {

  networking.firewall = {
    allowedTCPPorts = [
+      sshPort # ssh
      80
      443
      3306  # mysql
-      11012 # gitea ssh
      25565
      55228 # ss-bfs
    ];
@@ -199,13 +385,45 @@ in {

  systemd.tmpfiles.rules = [
    "d /var/www/store 0755 nginx nginx -"
+    "d ${worldOfSosalRoot} 0750 root nginx -"
+    "d ${worldOfSosalRoot}/releases 0750 root nginx -"
  ];

+  systemd.services.${giteaRunnerTokenEnvService} = {
+    description = "Prepare local Gitea Actions runner token environment";
+    requiredBy  = [ "${giteaRunnerService}.service" ];
+    before      = [ "${giteaRunnerService}.service" ];
+    serviceConfig = {
+      Type             = "oneshot";
+      RemainAfterExit  = true;
+      RuntimeDirectory = "gitea-runner-${giteaRunnerInstance}";
+      RuntimeDirectoryMode = "0700";
+    };
+    script = ''
+      set -eu
+      umask 077
+      token_file=${config.sops.secrets."gitea-runner/org-registration-token".path}
+      env_file=${giteaRunnerTokenEnv}
+
+      printf 'TOKEN=' > "$env_file"
+      tr -d '\n' < "$token_file" >> "$env_file"
+      printf '\n' >> "$env_file"
+    '';
+  };
+
+  systemd.services.${giteaRunnerService} = {
+    after = [
+      "gitea.service"
+      "${giteaRunnerTokenEnvService}.service"
+    ];
+    requires = [ "${giteaRunnerTokenEnvService}.service" ];
+  };
+
  services.nginx = {
    enable = true;
    # NOTE(yukkop): virtualHosts.${domain} is owned by the hectic-landing module
-    # (ACME-managed). See services.hectic-landing above.
-    virtualHosts."store.${domain}" = sslOpts // {
+    virtualHosts."store.${domain}" = {
+      enableACME = true;
      forceSSL = true;
      root = "/var/www/store";
      locations."/" = {
@@ -213,8 +431,72 @@ in {
          autoindex on;
        '';
      };
+      locations."= /world-of-sosal/" = {
+        extraConfig = ''
+          return 302 /world-of-sosal/index.html;
+        '';
+      };
+      locations."= /world-of-sosal/index.html" = {
+        extraConfig = ''
+          alias ${./static/world-of-sosal/index.html};
+          default_type text/html;
+          add_header Cache-Control "no-cache" always;
+          limit_except GET {
+            deny all;
+          }
+        '';
+      };
+      locations."= /world-of-sosal/latest.mrpack" = {
+        extraConfig = ''
+          root /var/www/store;
+          default_type application/zip;
+          add_header Content-Disposition "attachment" always;
+          add_header Cache-Control "no-cache, no-store, must-revalidate" always;
+          try_files $uri =404;
+          if ($request_method != GET) { return 405; }
+        '';
+      };
+      locations."= /world-of-sosal/SHA256SUMS" = {
+        extraConfig = ''
+          root /var/www/store;
+          default_type text/plain;
+          add_header Content-Disposition "attachment" always;
+          add_header Cache-Control "no-cache, no-store, must-revalidate" always;
+          try_files $uri =404;
+          if ($request_method != GET) { return 405; }
+        '';
+      };
+      locations."= /world-of-sosal/releases/" = {
+        extraConfig = ''
+          return 404;
+        '';
+      };
+      locations."~ ^/world-of-sosal/releases/[A-Za-z0-9][A-Za-z0-9._-]*\\.mrpack$" = {
+        extraConfig = ''
+          root /var/www/store;
+          default_type application/zip;
+          add_header Content-Disposition "attachment" always;
+          add_header Cache-Control "public, max-age=31536000, immutable" always;
+          try_files $uri =404;
+          if ($request_method != GET) { return 405; }
+        '';
+      };
+      locations."/world-of-sosal/" = {
+        extraConfig = ''
+          autoindex off;
+          limit_except GET {
+            deny all;
+          }
+          return 404;
+        '';
+      };
    };
-    virtualHosts."snuff.${domain}" = sslOpts // {
+    virtualHosts."lessons.${domain}" = {
+      enableACME = true;
+      forceSSL = true;
+    };
+    virtualHosts."snuff.${domain}" = {
+      enableACME = true;
      forceSSL = true;
      locations."/" = {
        extraConfig = ''
@@ -223,7 +505,8 @@ in {
        '';
      };
    };
-    virtualHosts."nrv.${domain}" = sslOpts // {
+    virtualHosts."nrv.${domain}" = {
+      enableACME = true;
      forceSSL = true;
      locations."/" = {
        extraConfig = ''
@@ -232,7 +515,8 @@ in {
        '';
      };
    };
-    virtualHosts."yukkop.${domain}" = sslOpts // {
+    virtualHosts."yukkop.${domain}" = {
+      enableACME = true;
      forceSSL = true;
      locations."/" = {
        extraConfig = ''
@@ -244,10 +528,15 @@ in {
    virtualHosts."gitea.${domain}" = {
      enableACME = true;
      forceSSL = true;
+      # NOTE(yukkop): allow large git pushes over HTTPS.
+      extraConfig = "client_max_body_size 512m;";
      locations."/" = {
        extraConfig = ''
          proxy_pass     http://127.0.0.1:11011/;
          proxy_redirect off;
+          proxy_set_header Host $host;
+          proxy_set_header X-Forwarded-Proto $scheme;
+          proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
        '';
      };
    };
@@ -257,12 +546,21 @@ in {
    gitea = {
      enable = true;
      package = pkgs.hectic.gitea-heatmap;
-      settings.service.DISABLE_REGISTRATION = false;
+      # Keep LFS storage limited to accounts provisioned by administrators.
+      settings.service.DISABLE_REGISTRATION = true;
+      settings.session.COOKIE_SECURE = true;
+      settings.actions.ENABLED = true;
+      # Long CUDA builds must not hit Gitea's default three-hour task watchdog.
+      settings.actions.ENDLESS_TASK_TIMEOUT = "8h";
      settings.server = {
+        HTTP_ADDR  = "127.0.0.1";
        HTTP_PORT  = 11011;
-        #SSH_PORT   = 22;
+        ROOT_URL   = "https://gitea.${domain}/";
+        SSH_PORT   = sshPort;
        SSH_DOMAIN = "hectic-lab.com";
      };
+      lfs.enable = true;
+      settings.lfs.LFS_MAX_FILE_SIZE = 536870912;
      database = {
        createDatabase = true;
        type = "postgres";
@@ -271,5 +569,27 @@ in {
        name = "gitea";
      };
    };
+    gitea-actions-runner.instances.${giteaRunnerInstance} = {
+      enable    = false;
+      name      = giteaRunnerInstance;
+      url       = "https://gitea.${domain}";
+      tokenFile = giteaRunnerTokenEnv;
+      labels = [
+        "nix:host"
+        "native:host"
+      ];
+      hostPackages = with pkgs; [
+        bash
+        cacert
+        coreutils
+        curl
+        git
+        gnutar
+        gzip
+        nix
+        nodejs
+        xz
+      ];
+    };
  };
 }
@@ -0,0 +1,16 @@
+{ domain, ... }:
+{
+  config,
+  ...
+}:
+{
+  hectic.services.immich = {
+    enable = true;
+    domain = "immich.${domain}";
+
+    storageBox = {
+      enable = true;
+      credentialsFile = config.sops.secrets."immich/storage-box".path;
+    };
+  };
+}
@@ -1,7 +1,6 @@
 {
  inputs,
  domain,
-  sslOpts,
  ...
 }: {
  pkgs,
@@ -29,7 +28,8 @@ in {
    enable = true;
  };

-  services.nginx.virtualHosts."${mechDomain}" = sslOpts // {
+  services.nginx.virtualHosts."${mechDomain}" = {
+    enableACME = true;
    forceSSL = true;
    root = inputs.mechabellum-replay-analysis.packages.${system}.frontend;

@@ -37,6 +37,7 @@ in {
      proxyPass = "http://${apiHost}:${builtins.toString apiPort}";
      extraConfig = ''
        proxy_http_version 1.1;
+        client_max_body_size 500M;
      '';
    };

@@ -0,0 +1,8 @@
+{ ... }:
+{
+  imports = [ (import ../../module/generic/minecraft-public-relay.nix { }) ];
+  services.minecraft-public-relay = {
+    enable = true;
+    publicKey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIKNWWegOVTOF3EOmam32iP7sMybULMTxsXuC+cEGITQ8 minecraft-wow-relay";
+  };
+}
@@ -3,7 +3,6 @@
  flake,
  self,
  domain,
-  sslOpts,
  ...
 }: { ... }: {
  hectic.services."sentinèlla" = {
@@ -17,7 +16,8 @@
  };

  services.nginx = {
-    virtualHosts."probe.${domain}" = sslOpts // {
+    virtualHosts."probe.${domain}" = {
+      enableACME = true;
      forceSSL = true;
      locations."/" = {
        proxyPass = "http://127.0.0.1:5988";
@@ -0,0 +1,18 @@
+<!doctype html>
+<html lang="en">
+<head>
+  <meta charset="utf-8">
+  <meta name="viewport" content="width=device-width, initial-scale=1">
+  <title>WorldOfSosal</title>
+</head>
+<body>
+  <main>
+    <h1>WorldOfSosal</h1>
+    <p><a href="https://store.hectic-lab.com/minecraft/world-of-sosal/">Install in Prism with automatic mod updates</a></p>
+    <p><a href="https://store.hectic-lab.com/world-of-sosal/latest.mrpack">Download latest pack</a></p>
+    <p><a href="prismlauncher://import?url=https%3A%2F%2Fstore.hectic-lab.com%2Fworld-of-sosal%2Flatest.mrpack">Import latest pack in Prism Launcher</a></p>
+    <p><a href="https://store.hectic-lab.com/world-of-sosal/SHA256SUMS">SHA-256 checksums</a></p>
+    <p>Updates are manual. Packs imported from arbitrary URLs do not update automatically.</p>
+  </main>
+</body>
+</html>
@@ -0,0 +1,52 @@
+# WorldOfSosal pack publishing
+
+The public endpoint is `https://store.hectic-lab.com/world-of-sosal/`. Nix
+deploys only its landing page and nginx configuration. Pack files, the checksum
+manifest, and `latest.mrpack` stay under `/var/www/store/world-of-sosal` on the
+host and never enter Git or the Nix store.
+
+## Publish an uploaded pack
+
+Run these commands on `hectic-lab` as root after the Storage Box pack has
+already been uploaded to a local staging path. Pick a stable version name; do
+not replace an existing versioned release.
+
+```sh
+set -eu
+source_pack=/path/to/already-uploaded/WorldOfSosal.mrpack
+version=2026-09-16
+root=/var/www/store/world-of-sosal
+release_name="WorldOfSosal-${version}.mrpack"
+release_path="$root/releases/$release_name"
+
+printf '%s  %s\n' \
+  f8c18acb9208e4592725632ae50dab4f9c308483b34fd43a6507c74fdbf8169f \
+  "$source_pack" | sha256sum --check --status
+test ! -e "$release_path"
+install -o root -g nginx -m 0640 "$source_pack" "$release_path.new"
+mv -T "$release_path.new" "$release_path"
+
+manifest="$root/.SHA256SUMS.$$"
+(cd "$root/releases" && sha256sum -- *.mrpack) > "$manifest"
+chown root:nginx "$manifest"
+chmod 0640 "$manifest"
+mv -Tf "$manifest" "$root/SHA256SUMS"
+
+latest="$root/.latest.mrpack.$$"
+ln -s "releases/$release_name" "$latest"
+mv -Tf "$latest" "$root/latest.mrpack"
+```
+
+Versioned releases use a one-year immutable cache policy. `latest.mrpack` and
+`SHA256SUMS` disable caching so an atomic replacement becomes visible quickly.
+The manifest is available at
+`https://store.hectic-lab.com/world-of-sosal/SHA256SUMS`.
+
+Import the current pack in Prism Launcher with:
+
+```text
+prismlauncher://import?url=https%3A%2F%2Fstore.hectic-lab.com%2Fworld-of-sosal%2Flatest.mrpack
+```
+
+Direct URL imports do not auto-update. Repeat the publication and import steps
+for each new pack version.
@@ -1,3 +1,17 @@
+# Current Minecraft access (2026-09-18)
+
+WorldOfSosal on the WoW map uses `store.hectic-lab.com:25568` publicly.
+The game server is `neuro:25567`; a restricted persistent reverse SSH tunnel
+connects it to the public relay. See `docs/minecraft-prism.md` and the Nix modules
+`minecraft/public-tunnel.nix` / `hectic-lab/minecraft-wow-proxy.nix`.
+
+Verified LAN: neuro is `192.168.88.10`, gateway `192.168.88.1`.
+SSH access is `95.31.254.84:34457`. Direct external Minecraft TCP probes timed out,
+and no UPnP IGD was discovered. The old TP-Link network and manual port-forward
+instructions below describe the previous network, not the active configuration.
+
+---
+
 # Router Access (TP-Link)

 The server `neuro` is behind a NAT router at `192.168.0.1`.
@@ -50,6 +64,8 @@ Ports that need to be forwarded from router to `192.168.0.10`:
 | 5269 | 5269 | TCP | XMPP (s2s) |
 | 10000 | 10000 | UDP | Jitsi Videobridge |
 | 25565 | 25565 | TCP | Minecraft |
+| 25567 | 25567 | TCP | Minecraft WoW Mine map |
+| 25568 | 25568 | TCP | Minecraft World of Sosal |

 ## Troubleshooting

@@ -15,32 +15,14 @@ in self.lib.nixpkgs-lib.nixosSystem {
      self.overlays.default
      inputs.nix-minecraft.overlay
    ];
-    config.allowUnfreePredicate = pkg: builtins.elem (self.lib.nixpkgs-lib.getName pkg) [
-      "minecraft-server"
-      "neoforge"
+    config.allowUnfreePredicate = pkg:
+      self.lib.cudaUnfreePredicate pkg || builtins.elem (self.lib.nixpkgs-lib.getName pkg) [
+       "minecraft-server"
+       "neoforge"
+       "steamcmd"
+       "steam-unwrapped"

-      "nvidia-x11"
-
-      "cuda_nvcc"
-      "cuda_cudart"
-      "cuda_cuobjdump"
-      "cuda_cupti"
-      "cuda_nvdisasm"
-      "cuda_cccl"
-      "cuda_nvml_dev"
-      "cuda_nvrtc"
-      "cuda_nvtx"
-      "cuda_profiler_api"
-
-      "libcusparse_lt"
-      "libcublas"
-      "libcufft"
-      "libcufile"
-      "libcurand"
-      "libcusolver"
-      "libnvjitlink"
-      "libcusparse"
-      "cudnn"
+       "nvidia-x11"
    ];
    # jitsi-meet depends on libolm which is marked insecure (CVE-2024-4519x)
    config.permittedInsecurePackages = [
@@ -1,61 +0,0 @@
-{
-  pkgs,
-  ...
-}: 
-{
-  services.minecraft-servers = {
-    enable = true;
-    eula = true;
-    openFirewall = true;
-
-    servers = {
-      vanilla = {
-        enable = true;
-        jvmOpts = "-Xmx6G -Xms2G";
-        package = pkgs.minecraftServers.vanilla-1_21_11;
-
-        serverProperties = {
-          server-port = 25565;
-          difficulty = "hard";
-          online-mode = true;
-          view-distance = 32;
-          level-seed = "8306359138650378643";
-          pause-when-empty-seconds = 0;
-        };
-      };
-
-      createAeronautics = {
-        enable = true;
-        jvmOpts = "-Xmx8G -Xms2G";
-        package = pkgs.minecraftServers.neoforge-1_21_1;
-
-        symlinks = {
-          mods = pkgs.linkFarmFromDrvs "mods" (
-            builtins.attrValues {
-              Sable = pkgs.fetchurl {
-                url = "https://cdn.modrinth.com/data/T9PomCSv/versions/g8CObHcP/sable-neoforge-1.21.1-1.1.3.jar";
-                sha512 = "8180e214681c171c9e3b7fa307f7a92bd7de0b8125d671291425f04a4ba26b408758d8ea80a6386d8e73bb1e6b02caf3f20afb9b91ecedd48c37ed44363ac961";
-              };
-              Create = pkgs.fetchurl {
-                url = "https://cdn.modrinth.com/data/LNytGWDc/versions/UjX6dr61/create-1.21.1-6.0.10.jar";
-                sha512 = "11cc8fc049d2f67f6548c7abfada6b82a3adb5c7ca410a742de04bbca76e03862c518721b88d806f6e6d768a4d68531fdb903a85859b25d1484d550cc7bafd4b";
-              };
-              CreateAeronautics = pkgs.fetchurl {
-                url = "https://cdn.modrinth.com/data/oWaK0Q19/versions/1sv6OtSz/create-aeronautics-bundled-1.21.1-1.1.3.jar";
-                sha512 = "94831bc4702b3864524258fa0a73a50ab3cd37e9c157b5c6688a6845b866ec5838452804050b55e490549d91dad909fc37f0d619f354c5676e2e2651b9c15ec6";
-              };
-            }
-          );
-        };
-
-        serverProperties = {
-          server-port = 25566;
-          difficulty = "hard";
-          online-mode = true;
-          view-distance = 20;
-          pause-when-empty-seconds = 0;
-        };
-      };
-    };
-  };
-}
@@ -0,0 +1,21 @@
+{ pkgs, ... }:
+
+{
+  services.minecraft-servers.servers.createAeronautics = {
+    enable = true;
+    jvmOpts = "-Xmx8G -Xms2G";
+    package = pkgs.minecraftServers.neoforge-1_21_1;
+
+    symlinks = {
+      mods = import ./mods.nix { inherit pkgs; };
+    };
+
+    serverProperties = {
+      server-port = 25566;
+      difficulty = "hard";
+      online-mode = true;
+      view-distance = 20;
+      pause-when-empty-seconds = 0;
+    };
+  };
+}
@@ -0,0 +1,17 @@
+{
+  imports = [
+    ./vanilla.nix
+    ./create-aeronautics.nix
+    ./wow-mine-map.nix
+    ./world-of-sosal.nix
+    ./public-tunnel.nix
+    ./world-import.nix
+    ./modpack-import.nix
+  ];
+
+  services.minecraft-servers = {
+    enable = true;
+    eula = true;
+    openFirewall = true;
+  };
+}
@@ -0,0 +1,460 @@
+{
+  config,
+  lib,
+  pkgs,
+  ...
+}: let
+  inherit (lib) mkOption types;
+
+  cfg = config.services.minecraft-modpack-imports;
+  enabledImports = lib.filterAttrs (_: import: import.enable) cfg;
+  dataDir = config.services.minecraft-servers.dataDir;
+  minecraftServers = config.services.minecraft-servers.servers;
+  targetServers = lib.mapAttrsToList (_: import: import.serverName) enabledImports;
+
+  importerUser = name: let
+    descriptiveName = "mc-pack-${name}";
+  in
+    if builtins.stringLength descriptiveName <= 31
+    then descriptiveName
+    else "mc-pack-${builtins.substring 0 16 (builtins.hashString "sha256" name)}";
+
+  stateDirectory = cacheDir:
+    if lib.hasPrefix "/var/lib/" cacheDir
+    then lib.removePrefix "/var/lib/" cacheDir
+    else null;
+
+  stateDirectoryCompatible = cacheDir: let
+    relative = stateDirectory cacheDir;
+    components = lib.splitString "/" (if relative == null then "" else relative);
+  in
+    relative != null
+    && relative != ""
+    && lib.all (component: component != "" && component != "." && component != "..") components;
+
+  escapeSftp = value:
+    "\"${lib.replaceStrings ["\\" "\""] ["\\\\" "\\\""] value}\"";
+
+  escapeTmpfiles = value:
+    lib.replaceStrings ["%" " " "\t"] ["%%" "\\x20" "\\x09"] value;
+
+  importerServices = lib.mkMerge (lib.mapAttrsToList (name: import: let
+    user = importerUser name;
+    unitName = "minecraft-modpack-import-${name}";
+    serverUnit = "minecraft-server-${import.serverName}.service";
+    cacheStateDirectory = stateDirectory import.cacheDir;
+    serverDir = "${dataDir}/${import.serverName}";
+  in {
+    ${unitName} = {
+      description = "Import Minecraft Modrinth pack ${name}";
+      before = [ serverUnit ];
+      wants = [ "network-online.target" ];
+      after = [ "network-online.target" ];
+      path = [
+        pkgs.coreutils
+        pkgs.curl
+        pkgs.findutils
+        pkgs.jq
+        pkgs.openssh
+        pkgs.unar
+      ];
+      serviceConfig = {
+        Type = "oneshot";
+        User = user;
+        Group = user;
+        RemainAfterExit = true;
+        TimeoutStartSec = import.timeout;
+        ProtectSystem = "strict";
+        ProtectHome = true;
+        PrivateDevices = true;
+        PrivateTmp = true;
+        ProtectKernelTunables = true;
+        ProtectKernelModules = true;
+        ProtectKernelLogs = true;
+        ProtectControlGroups = true;
+        NoNewPrivileges = true;
+        RestrictSUIDSGID = true;
+        LockPersonality = true;
+        CapabilityBoundingSet = [ "" ];
+        ReadWritePaths = [ import.cacheDir serverDir ];
+        UMask = "0007";
+      } // lib.optionalAttrs (stateDirectoryCompatible import.cacheDir) {
+        StateDirectory = cacheStateDirectory;
+      };
+      script = ''
+        set -eu
+        umask 007
+
+        cache_dir=${lib.escapeShellArg import.cacheDir}
+        server_dir=${lib.escapeShellArg serverDir}
+        archive_name=${lib.escapeShellArg import.archiveName}
+        archive="$cache_dir/$archive_name"
+        temporary_archive="$cache_dir/.$archive_name.$$"
+        extraction_dir="$cache_dir/.extract-${name}.$$"
+        staging_dir="$cache_dir/.stage-${name}.$$"
+        managed_paths="$cache_dir/managed-paths"
+        new_managed_paths="$cache_dir/.managed-paths.$$"
+        key=${lib.escapeShellArg import.sshKeyFile}
+
+        cleanup() {
+          rm -f "$temporary_archive" "$new_managed_paths"
+          rm -rf "$extraction_dir" "$staging_dir"
+        }
+        trap cleanup EXIT
+
+        safe_relative_path() {
+          case "$1" in
+            ""|/*|*\\*|.|..|./*|../*|*/./*|*/../*|*/.|*/..)
+              return 1
+              ;;
+          esac
+          return 0
+        }
+
+        archive_valid() {
+          [ -f "$archive" ] && printf '%s  %s\n' \
+            ${lib.escapeShellArg import.archiveSha256} \
+            "$archive" | sha256sum -c --status
+        }
+
+        archive_entries_valid() {
+          lsar -json "$archive" | jq -e '
+            (.lsarContents // .entries) as $entries
+            | ($entries | type == "array")
+            and ($entries | all(.[];
+                (.XADFileName // .XADPath) as $path
+                | ($path | type == "string")
+                and ($path | startswith("/") | not)
+                and ($path | contains("\\") | not)
+                and ($path | test("[[:cntrl:]]") | not)
+                and ([$path | split("/")[] | select(. == "" or . == "." or . == "..")] | length == 0)
+                and ((.XADIsSymbolicLink // false) | not)
+                and ((.XADIsHardLink // false) | not)
+                and ((.XADIsDevice // false) | not)
+                and ((.XADIsFIFO // false) | not)
+                and ((.XADIsSocket // false) | not)
+              )
+            )
+          ' >/dev/null
+        }
+
+        mkdir -p "$cache_dir" "$server_dir"
+        chmod 0700 "$cache_dir"
+
+        if ! archive_valid; then
+          rm -f "$archive"
+          downloaded=false
+          attempt=1
+          while [ "$attempt" -le ${toString import.retries} ]; do
+            rm -f "$temporary_archive"
+            if sftp \
+              -o BatchMode=yes \
+              -o StrictHostKeyChecking=yes \
+              -o UserKnownHostsFile=/etc/ssh/ssh_known_hosts \
+              -i "$key" \
+              -b - \
+              ${lib.escapeShellArg "${import.remoteUser}@${import.remoteHost}"} <<EOF
+        get ${escapeSftp import.remotePath} "$temporary_archive"
+        EOF
+            then
+              if printf '%s  %s\n' \
+                  ${lib.escapeShellArg import.archiveSha256} \
+                  "$temporary_archive" | sha256sum -c --status; then
+                mv "$temporary_archive" "$archive"
+                downloaded=true
+                break
+              fi
+            fi
+            rm -f "$temporary_archive"
+            attempt=$((attempt + 1))
+          done
+          if [ "$downloaded" != true ]; then
+            echo "Unable to download verified Minecraft modpack ${name}" >&2
+            exit 1
+          fi
+        fi
+
+        if ! archive_entries_valid; then
+          echo "Modpack archive contains unsafe entries" >&2
+          exit 1
+        fi
+
+        mkdir -p "$extraction_dir" "$staging_dir"
+        unar -quiet -output-directory "$extraction_dir" "$archive"
+
+        find "$extraction_dir" \
+          \( -type l -o -type b -o -type c -o -type p -o -type s \) \
+          -delete
+
+        manifest=$(find "$extraction_dir" -type f -name modrinth.index.json -print)
+        if [ -z "$manifest" ] || [ "$(printf '%s\n' "$manifest" | wc -l)" -ne 1 ]; then
+          echo "Modpack must contain exactly one modrinth.index.json" >&2
+          exit 1
+        fi
+        pack_root=$(dirname "$manifest")
+
+        if ! jq -e \
+          --argjson expectedDependencies ${lib.escapeShellArg (builtins.toJSON import.expectedDependencies)} \
+          '
+          . as $manifest
+          | .formatVersion == 1
+          and ($expectedDependencies | to_entries | all(.[];
+            $manifest.dependencies[.key] == .value
+          ))
+          and (.files | type == "array")
+          and all(.files[];
+            ((.env.server // "required") == "unsupported")
+            or (
+              (.path | type == "string")
+              and (.path | length > 0)
+              and (.path | startswith("mods/"))
+              and (.path | startswith("/") | not)
+              and (.path | contains("\\") | not)
+              and (.path | test("[[:cntrl:]]") | not)
+              and ([.path | split("/")[] | select(. == "" or . == "." or . == "..")] | length == 0)
+              and (.hashes.sha512 | type == "string")
+              and (.hashes.sha512 | test("^[0-9a-fA-F]{128}$"))
+              and (.downloads | type == "array")
+              and (.downloads | length > 0)
+              and (.downloads[0] | type == "string")
+              and (.downloads[0] | startswith("https://"))
+              and (.downloads[0] | test("[[:cntrl:]]") | not)
+            )
+          )
+        ' "$manifest" >/dev/null; then
+          echo "Modpack manifest contains unsafe or invalid server files" >&2
+          exit 1
+        fi
+
+        : > "$new_managed_paths"
+        jq -r '
+          .files[]
+          | select((.env.server // "required") != "unsupported")
+          | [.path, .hashes.sha512, .downloads[0]]
+          | @tsv
+        ' "$manifest" |
+        while IFS="$(printf '\t')" read -r relative expected_hash url; do
+          safe_relative_path "$relative" || exit 1
+          destination="$staging_dir/$relative"
+          mkdir -p "$(dirname "$destination")"
+          curl --fail --location --silent --show-error \
+            --retry ${toString import.retries} \
+            --output "$destination" \
+            "$url"
+          if ! printf '%s  %s\n' "$expected_hash" "$destination" |
+              sha512sum -c --status; then
+            echo "SHA-512 mismatch for $relative" >&2
+            exit 1
+          fi
+          printf '%s\n' "$relative" >> "$new_managed_paths"
+        done
+
+        overrides_dir="$pack_root/overrides"
+        if [ -d "$overrides_dir" ]; then
+          find "$overrides_dir" \
+            \( -type l -o -type b -o -type c -o -type p -o -type s \) \
+            -delete
+          cp -R "$overrides_dir/." "$staging_dir/"
+          find "$overrides_dir" -type f -printf '%P\n' |
+          while IFS= read -r relative; do
+            safe_relative_path "$relative" || exit 1
+            printf '%s\n' "$relative"
+          done >> "$new_managed_paths"
+        fi
+
+        # Nix Minecraft manages eula.txt via a symlink. Only reject symlinks
+        # in destinations we actually touch, including their parent directories.
+        safe_target_path() {
+          safe_relative_path "$1" || return 1
+          target="$server_dir/$1"
+          while [ "$target" != "$server_dir" ]; do
+            if [ -L "$target" ]; then
+              echo "Modpack destination contains symlink: $target" >&2
+              return 1
+            fi
+            target=$(dirname "$target")
+          done
+        }
+        while IFS= read -r relative; do
+          safe_target_path "$relative" || exit 1
+        done < "$new_managed_paths"
+        if [ -f "$managed_paths" ]; then
+          while IFS= read -r relative; do
+            safe_target_path "$relative" || exit 1
+          done < "$managed_paths"
+        fi
+
+        while IFS= read -r relative; do
+          safe_relative_path "$relative" || exit 1
+          source_file="$staging_dir/$relative"
+          target_file="$server_dir/$relative"
+          install -d -m 0770 -g minecraft "$(dirname "$target_file")"
+          install -m 0660 -g minecraft "$source_file" "$target_file"
+        done < "$new_managed_paths"
+
+        if [ -f "$managed_paths" ]; then
+          while IFS= read -r old_relative; do
+            safe_relative_path "$old_relative" || {
+              echo "Unsafe path in previous managed-paths file" >&2
+              exit 1
+            }
+            keep=false
+            while IFS= read -r relative; do
+              if [ "$old_relative" = "$relative" ]; then
+                keep=true
+                break
+              fi
+            done < "$new_managed_paths"
+            if [ "$keep" != true ]; then
+              rm -f "$server_dir/$old_relative"
+            fi
+          done < "$managed_paths"
+        fi
+
+        mv "$new_managed_paths" "$managed_paths"
+      '';
+    };
+
+    "minecraft-server-${import.serverName}" = {
+      requires = [ "${unitName}.service" ];
+      after = [ "${unitName}.service" ];
+    };
+  }) enabledImports);
+in {
+  options.services.minecraft-modpack-imports = mkOption {
+    default = { };
+    type = types.attrsOf (types.submodule ({ name, ... }: {
+      options = {
+        enable = lib.mkEnableOption "Minecraft Modrinth pack import ${name}";
+
+        serverName = mkOption {
+          type = types.str;
+          description = "minecraft-servers server receiving imported pack";
+        };
+
+        remoteHost = mkOption {
+          type = types.str;
+          description = "SSH host serving Modrinth pack archive";
+        };
+
+        remoteUser = mkOption {
+          type = types.str;
+          description = "SSH user used to download Modrinth pack archive";
+        };
+
+        remotePath = mkOption {
+          type = types.str;
+          description = "Remote path to Modrinth pack archive";
+        };
+
+        archiveName = mkOption {
+          type = types.str;
+          description = "Archive file name inside cache directory";
+        };
+
+        cacheDir = mkOption {
+          type = types.str;
+          default = "/var/lib/minecraft-modpacks/${name}";
+          description = "Persistent Modrinth archive and importer state directory";
+        };
+
+        archiveSha256 = mkOption {
+          type = types.strMatching "[0-9a-fA-F]{64}";
+          description = "Expected SHA-256 digest of Modrinth pack archive";
+        };
+
+        expectedDependencies = mkOption {
+          type = types.attrsOf types.str;
+          default = { };
+          description = "Required dependency versions in modrinth.index.json";
+        };
+
+        sshKeyFile = mkOption {
+          type = types.str;
+          description = "Runtime path to private SSH key";
+        };
+
+        hostPublicKey = mkOption {
+          type = types.str;
+          description = "Pinned SSH host public key";
+        };
+
+        retries = mkOption {
+          type = types.ints.positive;
+          default = 3;
+          description = "Maximum SFTP attempts and curl retry count";
+        };
+
+        timeout = mkOption {
+          type = types.str;
+          default = "30min";
+          description = "Importer service start timeout";
+        };
+      };
+    }));
+    description = "Modrinth packs imported before selected Minecraft servers start";
+  };
+
+  config = lib.mkIf (enabledImports != { }) {
+    assertions = lib.flatten (lib.mapAttrsToList (name: import: [
+      {
+        assertion = builtins.match "[A-Za-z0-9_-]+" name != null;
+        message = "services.minecraft-modpack-imports.${name}: name must contain only letters, digits, underscores, or hyphens";
+      }
+      {
+        assertion = builtins.match "/.*" import.cacheDir != null;
+        message = "services.minecraft-modpack-imports.${name}.cacheDir must be absolute";
+      }
+      {
+        assertion = builtins.match "/var/lib(/[A-Za-z0-9][A-Za-z0-9._-]*)+" import.cacheDir != null;
+        message = "services.minecraft-modpack-imports.${name}.cacheDir must be beneath /var/lib with safe path components";
+      }
+      {
+        assertion = builtins.match "[A-Za-z0-9_-]+" import.serverName != null;
+        message = "services.minecraft-modpack-imports.${name}.serverName must contain only letters, digits, underscores, or hyphens";
+      }
+      {
+        assertion = !lib.hasInfix "\n" import.remotePath && !lib.hasInfix "\r" import.remotePath;
+        message = "services.minecraft-modpack-imports.${name}.remotePath must not contain newlines";
+      }
+      {
+        assertion = builtins.hasAttr import.serverName minecraftServers
+          && (builtins.getAttr import.serverName minecraftServers).enable;
+        message = "services.minecraft-modpack-imports.${name}.serverName must name an enabled Minecraft server";
+      }
+      {
+        assertion = builtins.match "[A-Za-z0-9._-]+" import.archiveName != null
+          && import.archiveName != "."
+          && import.archiveName != ".."
+          && import.archiveName != "managed-paths";
+        message = "services.minecraft-modpack-imports.${name}.archiveName must be a file name";
+      }
+      {
+        assertion = lib.length (lib.unique targetServers) == lib.length targetServers;
+        message = "services.minecraft-modpack-imports: each server target must be unique";
+      }
+    ]) enabledImports);
+
+    users.groups = lib.mapAttrs' (name: _: lib.nameValuePair (importerUser name) { }) enabledImports;
+    users.users = lib.mapAttrs' (name: _: let
+      user = importerUser name;
+    in lib.nameValuePair user {
+      description = "Minecraft modpack importer ${name}";
+      isSystemUser = true;
+      group = user;
+      extraGroups = [ "minecraft" ];
+    }) enabledImports;
+
+    programs.ssh.knownHosts = lib.mapAttrs' (name: import:
+      lib.nameValuePair "minecraft-modpack-import-${name}" {
+        hostNames = [ import.remoteHost ];
+        publicKey = import.hostPublicKey;
+      }) enabledImports;
+
+    systemd.tmpfiles.rules = lib.flatten (lib.mapAttrsToList (name: import:
+      lib.optional (!stateDirectoryCompatible import.cacheDir)
+        "d ${escapeTmpfiles import.cacheDir} 0700 ${importerUser name} ${importerUser name} -") enabledImports);
+
+    systemd.services = importerServices;
+  };
+}
@@ -0,0 +1,18 @@
+{ pkgs }:
+
+pkgs.linkFarmFromDrvs "create-aeronautics-mods" (
+  builtins.attrValues {
+    Sable = pkgs.fetchurl {
+      url = "https://cdn.modrinth.com/data/T9PomCSv/versions/g8CObHcP/sable-neoforge-1.21.1-1.1.3.jar";
+      sha512 = "8180e214681c171c9e3b7fa307f7a92bd7de0b8125d671291425f04a4ba26b408758d8ea80a6386d8e73bb1e6b02caf3f20afb9b91ecedd48c37ed44363ac961";
+    };
+    Create = pkgs.fetchurl {
+      url = "https://cdn.modrinth.com/data/LNytGWDc/versions/UjX6dr61/create-1.21.1-6.0.10.jar";
+      sha512 = "11cc8fc049d2f67f6548c7abfada6b82a3adb5c7ca410a742de04bbca76e03862c518721b88d806f6e6d768a4d68531fdb903a85859b25d1484d550cc7bafd4b";
+    };
+    CreateAeronautics = pkgs.fetchurl {
+      url = "https://cdn.modrinth.com/data/oWaK0Q19/versions/1sv6OtSz/create-aeronautics-bundled-1.21.1-1.1.3.jar";
+      sha512 = "94831bc4702b3864524258fa0a73a50ab3cd37e9c157b5c6688a6845b866ec5838452804050b55e490549d91dad909fc37f0d619f354c5676e2e2651b9c15ec6";
+    };
+  }
+)
@@ -0,0 +1,363 @@
+{
+  "build": {
+    "version": "21.1.250",
+    "src": {
+      "url": "https://maven.neoforged.net/releases/net/neoforged/neoforge/21.1.250/neoforge-21.1.250-installer.jar",
+      "hash": "sha256-DkepG6ITmo20v3Ynrwgfe1eJtQi7A57o3qEnK3lpPWA="
+    },
+    "libraries": [
+      "net.neoforged.fancymodloader:earlydisplay:4.0.44",
+      "net.neoforged.fancymodloader:loader:4.0.44",
+      "net.neoforged.accesstransformers:at-modlauncher:10.0.1",
+      "net.neoforged:accesstransformers:10.0.1",
+      "net.neoforged:bus:8.0.5",
+      "net.neoforged:coremods:7.0.3",
+      "cpw.mods:modlauncher:11.0.5",
+      "net.neoforged:mergetool:2.0.0:api",
+      "com.electronwill.night-config:toml:3.8.3",
+      "com.electronwill.night-config:core:3.8.3",
+      "net.neoforged:JarJarSelector:0.4.1",
+      "net.neoforged:JarJarMetadata:0.4.1",
+      "org.apache.maven:maven-artifact:3.8.5",
+      "net.jodah:typetools:0.6.3",
+      "net.minecrell:terminalconsoleappender:1.3.0",
+      "net.fabricmc:sponge-mixin:0.15.2+mixin.0.8.7",
+      "org.openjdk.nashorn:nashorn-core:15.4",
+      "org.apache.commons:commons-lang3:3.14.0",
+      "cpw.mods:bootstraplauncher:2.0.2",
+      "cpw.mods:securejarhandler:3.0.8",
+      "org.ow2.asm:asm-commons:9.10.1",
+      "org.ow2.asm:asm-util:9.10.1",
+      "org.ow2.asm:asm-analysis:9.10.1",
+      "org.ow2.asm:asm-tree:9.10.1",
+      "org.ow2.asm:asm:9.10.1",
+      "net.neoforged:JarJarFileSystems:0.4.1",
+      "net.sf.jopt-simple:jopt-simple:5.0.4",
+      "org.slf4j:slf4j-api:2.0.9",
+      "org.antlr:antlr4-runtime:4.13.1",
+      "com.mojang:logging:1.2.7",
+      "org.apache.logging.log4j:log4j-slf4j2-impl:2.22.1",
+      "org.apache.logging.log4j:log4j-core:2.22.1",
+      "org.apache.logging.log4j:log4j-api:2.22.1",
+      "org.jline:jline-reader:3.20.0",
+      "org.jline:jline-terminal:3.20.0",
+      "commons-io:commons-io:2.15.1",
+      "net.minecraftforge:srgutils:0.4.15",
+      "com.google.guava:guava:32.1.2-jre",
+      "com.google.guava:failureaccess:1.0.1",
+      "com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava",
+      "com.google.code.findbugs:jsr305:3.0.2",
+      "org.checkerframework:checker-qual:3.33.0",
+      "com.google.errorprone:error_prone_annotations:2.18.0",
+      "com.google.j2objc:j2objc-annotations:2.8",
+      "com.google.code.gson:gson:2.10.1",
+      "org.codehaus.plexus:plexus-utils:3.3.0",
+      "com.machinezoo.noexception:noexception:1.7.1",
+      "net.neoforged:neoform:1.21.1-20240808.144430@zip",
+      "net.neoforged.installertools:binarypatcher:2.1.2:fatjar",
+      "net.neoforged:AutoRenamingTool:2.0.3:all",
+      "net.neoforged.installertools:installertools:2.1.2",
+      "net.neoforged:srgutils:1.0.0",
+      "net.md-5:SpecialSource:1.11.0",
+      "com.google.code.gson:gson:2.8.9",
+      "de.siegmar:fastcsv:2.0.0",
+      "org.ow2.asm:asm-commons:9.3",
+      "net.neoforged.installertools:cli-utils:2.1.2",
+      "com.google.guava:guava:20.0",
+      "com.opencsv:opencsv:4.4",
+      "org.ow2.asm:asm-analysis:9.3",
+      "org.ow2.asm:asm-tree:9.3",
+      "org.ow2.asm:asm:9.3",
+      "org.apache.commons:commons-text:1.3",
+      "org.apache.commons:commons-lang3:3.8.1",
+      "commons-beanutils:commons-beanutils:1.9.3",
+      "org.apache.commons:commons-collections4:4.2",
+      "commons-logging:commons-logging:1.2",
+      "commons-collections:commons-collections:3.2.2",
+      "net.neoforged.installertools:jarsplitter:2.1.2",
+      "net.neoforged:neoforge:21.1.250:universal"
+    ]
+  },
+  "libraryLocks": {
+    "net.neoforged.fancymodloader:earlydisplay:4.0.44": {
+      "url": "https://maven.neoforged.net/releases/net/neoforged/fancymodloader/earlydisplay/4.0.44/earlydisplay-4.0.44.jar",
+      "hash": "sha1-yyr88VZyGA7KHEmxybEgiVHvMi4="
+    },
+    "net.neoforged.fancymodloader:loader:4.0.44": {
+      "url": "https://maven.neoforged.net/releases/net/neoforged/fancymodloader/loader/4.0.44/loader-4.0.44.jar",
+      "hash": "sha1-Y0M2CLaDAqRC49uoeIDlT0LqBYM="
+    },
+    "net.neoforged.accesstransformers:at-modlauncher:10.0.1": {
+      "url": "https://maven.neoforged.net/releases/net/neoforged/accesstransformers/at-modlauncher/10.0.1/at-modlauncher-10.0.1.jar",
+      "hash": "sha1-WrpQICrO6tCGvAn7wnUcnwXLSJA="
+    },
+    "net.neoforged:accesstransformers:10.0.1": {
+      "url": "https://maven.neoforged.net/releases/net/neoforged/accesstransformers/10.0.1/accesstransformers-10.0.1.jar",
+      "hash": "sha1-/YO1cl926ukRXpNV+hxFampEFAA="
+    },
+    "net.neoforged:bus:8.0.5": {
+      "url": "https://maven.neoforged.net/releases/net/neoforged/bus/8.0.5/bus-8.0.5.jar",
+      "hash": "sha1-Wy0zKFq10VVOl5itmMQNbqOGi9U="
+    },
+    "net.neoforged:coremods:7.0.3": {
+      "url": "https://maven.neoforged.net/releases/net/neoforged/coremods/7.0.3/coremods-7.0.3.jar",
+      "hash": "sha1-CRR+b2OLQnKzvV/I+SrTeAJRLGw="
+    },
+    "cpw.mods:modlauncher:11.0.5": {
+      "url": "https://maven.neoforged.net/releases/cpw/mods/modlauncher/11.0.5/modlauncher-11.0.5.jar",
+      "hash": "sha1-uPDUkpT3M/22FzkxsmNVPpQ9yVA="
+    },
+    "net.neoforged:mergetool:2.0.0:api": {
+      "url": "https://maven.neoforged.net/releases/net/neoforged/mergetool/2.0.0/mergetool-2.0.0-api.jar",
+      "hash": "sha1-Uv4ZSb5k4zA6q6qiHjFfVR25yfQ="
+    },
+    "com.electronwill.night-config:toml:3.8.3": {
+      "url": "https://maven.neoforged.net/releases/com/electronwill/night-config/toml/3.8.3/toml-3.8.3.jar",
+      "hash": "sha1-kLL9bvy0p9W5gQz3j52CSZTXF/I="
+    },
+    "com.electronwill.night-config:core:3.8.3": {
+      "url": "https://maven.neoforged.net/releases/com/electronwill/night-config/core/3.8.3/core-3.8.3.jar",
+      "hash": "sha1-tEKpXwnjSZJ/WpRey1lEVYcPz08="
+    },
+    "net.neoforged:JarJarSelector:0.4.1": {
+      "url": "https://maven.neoforged.net/releases/net/neoforged/JarJarSelector/0.4.1/JarJarSelector-0.4.1.jar",
+      "hash": "sha1-+zzHpYryKtKICtuYr21RgSjEfa4="
+    },
+    "net.neoforged:JarJarMetadata:0.4.1": {
+      "url": "https://maven.neoforged.net/releases/net/neoforged/JarJarMetadata/0.4.1/JarJarMetadata-0.4.1.jar",
+      "hash": "sha1-+NoDaD3IFpRVbcPhd8Xju3eub8s="
+    },
+    "org.apache.maven:maven-artifact:3.8.5": {
+      "url": "https://maven.neoforged.net/releases/org/apache/maven/maven-artifact/3.8.5/maven-artifact-3.8.5.jar",
+      "hash": "sha1-RDP1DAfevvrtBVO9AGj09I1EkxM="
+    },
+    "net.jodah:typetools:0.6.3": {
+      "url": "https://maven.neoforged.net/releases/net/jodah/typetools/0.6.3/typetools-0.6.3.jar",
+      "hash": "sha1-oBqqbdrqnsB+xPIJSHt6RqUmKDo="
+    },
+    "net.minecrell:terminalconsoleappender:1.3.0": {
+      "url": "https://maven.neoforged.net/releases/net/minecrell/terminalconsoleappender/1.3.0/terminalconsoleappender-1.3.0.jar",
+      "hash": "sha1-tWLpu2EjXJUg4mKCze5x+PgC0fw="
+    },
+    "net.fabricmc:sponge-mixin:0.15.2+mixin.0.8.7": {
+      "url": "https://maven.neoforged.net/releases/net/fabricmc/sponge-mixin/0.15.2+mixin.0.8.7/sponge-mixin-0.15.2+mixin.0.8.7.jar",
+      "hash": "sha1-KvLwIdjgKgIg3CenpytGZtZtRMo="
+    },
+    "org.openjdk.nashorn:nashorn-core:15.4": {
+      "url": "https://maven.neoforged.net/releases/org/openjdk/nashorn/nashorn-core/15.4/nashorn-core-15.4.jar",
+      "hash": "sha1-9n9f+qX1Ewz2+5sTPaAMffO1MqU="
+    },
+    "org.apache.commons:commons-lang3:3.14.0": {
+      "url": "https://libraries.minecraft.net/org/apache/commons/commons-lang3/3.14.0/commons-lang3-3.14.0.jar",
+      "hash": "sha1-HtRxGUsC8sbLc0oM1vbxB8Zzr64="
+    },
+    "cpw.mods:bootstraplauncher:2.0.2": {
+      "url": "https://maven.neoforged.net/releases/cpw/mods/bootstraplauncher/2.0.2/bootstraplauncher-2.0.2.jar",
+      "hash": "sha1-Gi0HbLwzsFIMus1ZEiRCeyogBH0="
+    },
+    "cpw.mods:securejarhandler:3.0.8": {
+      "url": "https://maven.neoforged.net/releases/cpw/mods/securejarhandler/3.0.8/securejarhandler-3.0.8.jar",
+      "hash": "sha1-wO+Vzs2GmaBEkFOsfZwWB0jZAs0="
+    },
+    "org.ow2.asm:asm-commons:9.10.1": {
+      "url": "https://maven.neoforged.net/releases/org/ow2/asm/asm-commons/9.10.1/asm-commons-9.10.1.jar",
+      "hash": "sha1-QinkxV/Y4Bwj+f6YhAdcxiiqzFA="
+    },
+    "org.ow2.asm:asm-util:9.10.1": {
+      "url": "https://maven.neoforged.net/releases/org/ow2/asm/asm-util/9.10.1/asm-util-9.10.1.jar",
+      "hash": "sha1-e7nUUOjUy/n54ECWxEu/5/uoCxU="
+    },
+    "org.ow2.asm:asm-analysis:9.10.1": {
+      "url": "https://maven.neoforged.net/releases/org/ow2/asm/asm-analysis/9.10.1/asm-analysis-9.10.1.jar",
+      "hash": "sha1-jUnxTVH2Mssdh8iNHOr1DbDYrxs="
+    },
+    "org.ow2.asm:asm-tree:9.10.1": {
+      "url": "https://maven.neoforged.net/releases/org/ow2/asm/asm-tree/9.10.1/asm-tree-9.10.1.jar",
+      "hash": "sha1-4kQzKhdWTB0VckSTmahC3jWIG+I="
+    },
+    "org.ow2.asm:asm:9.10.1": {
+      "url": "https://maven.neoforged.net/releases/org/ow2/asm/asm/9.10.1/asm-9.10.1.jar",
+      "hash": "sha1-raIUHAzFLuj1xIzV+kzg55TyIjY="
+    },
+    "net.neoforged:JarJarFileSystems:0.4.1": {
+      "url": "https://maven.neoforged.net/releases/net/neoforged/JarJarFileSystems/0.4.1/JarJarFileSystems-0.4.1.jar",
+      "hash": "sha1-ePWfid780DLteIsVHKag1ArOeWo="
+    },
+    "net.sf.jopt-simple:jopt-simple:5.0.4": {
+      "url": "https://libraries.minecraft.net/net/sf/jopt-simple/jopt-simple/5.0.4/jopt-simple-5.0.4.jar",
+      "hash": "sha1-T9rC++kt+thqpukwFzb2tDQqP1w="
+    },
+    "org.slf4j:slf4j-api:2.0.9": {
+      "url": "https://libraries.minecraft.net/org/slf4j/slf4j-api/2.0.9/slf4j-api-2.0.9.jar",
+      "hash": "sha1-fPJyb9z7yGEPmnH7PtY5hx8xU0A="
+    },
+    "org.antlr:antlr4-runtime:4.13.1": {
+      "url": "https://maven.neoforged.net/releases/org/antlr/antlr4-runtime/4.13.1/antlr4-runtime-4.13.1.jar",
+      "hash": "sha1-FxJbrh2WViTiZe9JVS9kZaK/owc="
+    },
+    "com.mojang:logging:1.2.7": {
+      "url": "https://libraries.minecraft.net/com/mojang/logging/1.2.7/logging-1.2.7.jar",
+      "hash": "sha1-JMuV/7DjQz/W6ETATmgAnlBMocA="
+    },
+    "org.apache.logging.log4j:log4j-slf4j2-impl:2.22.1": {
+      "url": "https://libraries.minecraft.net/org/apache/logging/log4j/log4j-slf4j2-impl/2.22.1/log4j-slf4j2-impl-2.22.1.jar",
+      "hash": "sha1-1+ZpPCYGy35zNQR9e7lt7FLbVmU="
+    },
+    "org.apache.logging.log4j:log4j-core:2.22.1": {
+      "url": "https://libraries.minecraft.net/org/apache/logging/log4j/log4j-core/2.22.1/log4j-core-2.22.1.jar",
+      "hash": "sha1-cYOiVRCgKtAMxqldOz0qfTxajcQ="
+    },
+    "org.apache.logging.log4j:log4j-api:2.22.1": {
+      "url": "https://libraries.minecraft.net/org/apache/logging/log4j/log4j-api/2.22.1/log4j-api-2.22.1.jar",
+      "hash": "sha1-vqb+3mMo+rr9fmg2MWGn6mYFq9E="
+    },
+    "org.jline:jline-reader:3.20.0": {
+      "url": "https://maven.neoforged.net/releases/org/jline/jline-reader/3.20.0/jline-reader-3.20.0.jar",
+      "hash": "sha1-jxVBWwIqJbRz6OFsKK6RMYb/ucQ="
+    },
+    "org.jline:jline-terminal:3.20.0": {
+      "url": "https://maven.neoforged.net/releases/org/jline/jline-terminal/3.20.0/jline-terminal-3.20.0.jar",
+      "hash": "sha1-0N3McI3fUno0VMlBt7kiXMg6Ff8="
+    },
+    "commons-io:commons-io:2.15.1": {
+      "url": "https://libraries.minecraft.net/commons-io/commons-io/2.15.1/commons-io-2.15.1.jar",
+      "hash": "sha1-8RVg2hiatWOlyONRlBQVQw6TBOo="
+    },
+    "net.minecraftforge:srgutils:0.4.15": {
+      "url": "https://maven.neoforged.net/releases/net/minecraftforge/srgutils/0.4.15/srgutils-0.4.15.jar",
+      "hash": "sha1-ykCLExdZR48WTgEPrg1zmX4SX7U="
+    },
+    "com.google.guava:guava:32.1.2-jre": {
+      "url": "https://libraries.minecraft.net/com/google/guava/guava/32.1.2-jre/guava-32.1.2-jre.jar",
+      "hash": "sha1-XmTsfgVkVr7zpLxMb9rvceirYxg="
+    },
+    "com.google.guava:failureaccess:1.0.1": {
+      "url": "https://libraries.minecraft.net/com/google/guava/failureaccess/1.0.1/failureaccess-1.0.1.jar",
+      "hash": "sha1-Hc8d44Kgv5Wj2LCElUbIi6wSksk="
+    },
+    "com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava": {
+      "url": "https://libraries.minecraft.net/com/google/guava/listenablefuture/9999.0-empty-to-avoid-conflict-with-guava/listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar",
+      "hash": "sha1-tCFSbF8pcpWt7xyIblJGw51Kxik="
+    },
+    "com.google.code.findbugs:jsr305:3.0.2": {
+      "url": "https://libraries.minecraft.net/com/google/code/findbugs/jsr305/3.0.2/jsr305-3.0.2.jar",
+      "hash": "sha1-JeouiwwziodzE71GctP+BW6njw0="
+    },
+    "org.checkerframework:checker-qual:3.33.0": {
+      "url": "https://libraries.minecraft.net/org/checkerframework/checker-qual/3.33.0/checker-qual-3.33.0.jar",
+      "hash": "sha1-3itgti2kh2RPwR9zTnPIsLQxI48="
+    },
+    "com.google.errorprone:error_prone_annotations:2.18.0": {
+      "url": "https://libraries.minecraft.net/com/google/errorprone/error_prone_annotations/2.18.0/error_prone_annotations-2.18.0.jar",
+      "hash": "sha1-ibaEJXCW9Uj6Oaffn9qkCdTU35E="
+    },
+    "com.google.j2objc:j2objc-annotations:2.8": {
+      "url": "https://libraries.minecraft.net/com/google/j2objc/j2objc-annotations/2.8/j2objc-annotations-2.8.jar",
+      "hash": "sha1-yFJw4wfnuCLxCGuTaJEkuJdo4nM="
+    },
+    "com.google.code.gson:gson:2.10.1": {
+      "url": "https://libraries.minecraft.net/com/google/code/gson/gson/2.10.1/gson-2.10.1.jar",
+      "hash": "sha1-s63UeNQ4K3jqILFnE5CoWAAv62w="
+    },
+    "org.codehaus.plexus:plexus-utils:3.3.0": {
+      "url": "https://maven.neoforged.net/releases/org/codehaus/plexus/plexus-utils/3.3.0/plexus-utils-3.3.0.jar",
+      "hash": "sha1-z0O1OR3mI7Nv4GaiESe674LGQCI="
+    },
+    "com.machinezoo.noexception:noexception:1.7.1": {
+      "url": "https://maven.neoforged.net/releases/com/machinezoo/noexception/noexception/1.7.1/noexception-1.7.1.jar",
+      "hash": "sha1-tlMwyY44ofkV+lSm5eykllBePwo="
+    },
+    "net.neoforged:neoform:1.21.1-20240808.144430@zip": {
+      "url": "https://maven.neoforged.net/releases/net/neoforged/neoform/1.21.1-20240808.144430/neoform-1.21.1-20240808.144430.zip",
+      "hash": "sha1-gR4r2G+izaKBLl6OUdcY6ovW0/Q="
+    },
+    "net.neoforged.installertools:binarypatcher:2.1.2:fatjar": {
+      "url": "https://maven.neoforged.net/releases/net/neoforged/installertools/binarypatcher/2.1.2/binarypatcher-2.1.2-fatjar.jar",
+      "hash": "sha1-dZtj7zk+2AQY7B6k0jPNYVLQJjc="
+    },
+    "net.neoforged:AutoRenamingTool:2.0.3:all": {
+      "url": "https://maven.neoforged.net/releases/net/neoforged/AutoRenamingTool/2.0.3/AutoRenamingTool-2.0.3-all.jar",
+      "hash": "sha1-2YkMcbQ2b4hsKxAGeCBDpqaBbrY="
+    },
+    "net.neoforged.installertools:installertools:2.1.2": {
+      "url": "https://maven.neoforged.net/releases/net/neoforged/installertools/installertools/2.1.2/installertools-2.1.2.jar",
+      "hash": "sha1-clJMA2L4EtiqTNtMA+m0Xitxrjs="
+    },
+    "net.neoforged:srgutils:1.0.0": {
+      "url": "https://maven.neoforged.net/releases/net/neoforged/srgutils/1.0.0/srgutils-1.0.0.jar",
+      "hash": "sha1-uf5s2rSUmDIXy8FMxvksjmxhZSY="
+    },
+    "net.md-5:SpecialSource:1.11.0": {
+      "url": "https://maven.neoforged.net/releases/net/md-5/SpecialSource/1.11.0/SpecialSource-1.11.0.jar",
+      "hash": "sha1-Q7hMS7jQHPkKKd/uwclYpLZLr0Y="
+    },
+    "com.google.code.gson:gson:2.8.9": {
+      "url": "https://libraries.minecraft.net/com/google/code/gson/gson/2.8.9/gson-2.8.9.jar",
+      "hash": "sha1-ikMsHWgleB4hoC2y4sM8X94oM7k="
+    },
+    "de.siegmar:fastcsv:2.0.0": {
+      "url": "https://maven.neoforged.net/releases/de/siegmar/fastcsv/2.0.0/fastcsv-2.0.0.jar",
+      "hash": "sha1-thXybAPt6slmYYuTue5PTu1QquE="
+    },
+    "org.ow2.asm:asm-commons:9.3": {
+      "url": "https://maven.neoforged.net/releases/org/ow2/asm/asm-commons/9.3/asm-commons-9.3.jar",
+      "hash": "sha1-HypDLRIS9cNSrmB9e2HcriDCCvU="
+    },
+    "net.neoforged.installertools:cli-utils:2.1.2": {
+      "url": "https://maven.neoforged.net/releases/net/neoforged/installertools/cli-utils/2.1.2/cli-utils-2.1.2.jar",
+      "hash": "sha1-5aMRXrnimRF0cmdZRr6KcuSZCtY="
+    },
+    "com.google.guava:guava:20.0": {
+      "url": "https://maven.neoforged.net/releases/com/google/guava/guava/20.0/guava-20.0.jar",
+      "hash": "sha1-iVB3ASSTiOHtXdz4xB9M4b54Me8="
+    },
+    "com.opencsv:opencsv:4.4": {
+      "url": "https://maven.neoforged.net/releases/com/opencsv/opencsv/4.4/opencsv-4.4.jar",
+      "hash": "sha1-Ulkyoe30bJynWqnIPTHcGAwYaGU="
+    },
+    "org.ow2.asm:asm-analysis:9.3": {
+      "url": "https://maven.neoforged.net/releases/org/ow2/asm/asm-analysis/9.3/asm-analysis-9.3.jar",
+      "hash": "sha1-SwcfIRs3w44On1mYVQGXyFk/atg="
+    },
+    "org.ow2.asm:asm-tree:9.3": {
+      "url": "https://maven.neoforged.net/releases/org/ow2/asm/asm-tree/9.3/asm-tree-9.3.jar",
+      "hash": "sha1-eNLs1hMYtaWM0E+yN2NsDoa3fZc="
+    },
+    "org.ow2.asm:asm:9.3": {
+      "url": "https://libraries.minecraft.net/org/ow2/asm/asm/9.3/asm-9.3.jar",
+      "hash": "sha1-jmMA71HB2AGn7WLQfNIhrKOpBkA="
+    },
+    "org.apache.commons:commons-text:1.3": {
+      "url": "https://maven.neoforged.net/releases/org/apache/commons/commons-text/1.3/commons-text-1.3.jar",
+      "hash": "sha1-mr9hcIpmq15V9haaIA2/xYS1Rtk="
+    },
+    "org.apache.commons:commons-lang3:3.8.1": {
+      "url": "https://maven.neoforged.net/releases/org/apache/commons/commons-lang3/3.8.1/commons-lang3-3.8.1.jar",
+      "hash": "sha1-ZQWnKgl9knD3qee/QsQjgoMkd1U="
+    },
+    "commons-beanutils:commons-beanutils:1.9.3": {
+      "url": "https://maven.neoforged.net/releases/commons-beanutils/commons-beanutils/1.9.3/commons-beanutils-1.9.3.jar",
+      "hash": "sha1-yEVwPeM03ca0s80mg1RYyxy6Hz0="
+    },
+    "org.apache.commons:commons-collections4:4.2": {
+      "url": "https://maven.neoforged.net/releases/org/apache/commons/commons-collections4/4.2/commons-collections4-4.2.jar",
+      "hash": "sha1-VOvqCltlPTxoATHnP+gHu494xO0="
+    },
+    "commons-logging:commons-logging:1.2": {
+      "url": "https://libraries.minecraft.net/commons-logging/commons-logging/1.2/commons-logging-1.2.jar",
+      "hash": "sha1-S/wSrf5IQr8HtlfwNpxMtSKVVoY="
+    },
+    "commons-collections:commons-collections:3.2.2": {
+      "url": "https://maven.neoforged.net/releases/commons-collections/commons-collections/3.2.2/commons-collections-3.2.2.jar",
+      "hash": "sha1-itcv45+oyR6q8Sqtsh4MNmH+JtU="
+    },
+    "net.neoforged.installertools:jarsplitter:2.1.2": {
+      "url": "https://maven.neoforged.net/releases/net/neoforged/installertools/jarsplitter/2.1.2/jarsplitter-2.1.2.jar",
+      "hash": "sha1-inkWvgoOWJiXvqt8g5ByYxBn5I4="
+    },
+    "net.neoforged:neoforge:21.1.250:universal": {
+      "url": "https://maven.neoforged.net/releases/net/neoforged/neoforge/21.1.250/neoforge-21.1.250-universal.jar",
+      "hash": "sha1-IUxSk8hZmwqq39v6kkU7ojkYohk="
+    }
+  }
+}
@@ -0,0 +1,49 @@
+{ config, pkgs, ... }:
+let
+  mkTunnel = relay: {
+    description = "WorldOfSosal WoW reverse tunnel to ${relay.name}";
+    startLimitIntervalSec = 0;
+    wantedBy = [ "multi-user.target" ];
+    wants = [ "network-online.target" ];
+    after = [ "network-online.target" ];
+    serviceConfig = {
+      User = "mc-wow-tunnel";
+      Group = "mc-wow-tunnel";
+      ExecStart = "${pkgs.openssh}/bin/ssh -NT -i ${config.sops.secrets."minecraft/wow-tunnel-key".path} -o IPQoS=none -o Ciphers=aes256-ctr -o MACs=hmac-sha2-256-etm@openssh.com -o KexAlgorithms=curve25519-sha256 -o BatchMode=yes -o IdentitiesOnly=yes -o StrictHostKeyChecking=yes -o UserKnownHostsFile=/etc/ssh/ssh_known_hosts -o ExitOnForwardFailure=yes -o ServerAliveInterval=15 -o ServerAliveCountMax=3 -o ConnectTimeout=10 -R 127.0.0.1:25577:127.0.0.1:25567 mc-wow-relay@${relay.address}";
+      Restart = "always";
+      RestartSec = 10;
+      NoNewPrivileges = true;
+      ProtectSystem = "strict";
+      ProtectHome = true;
+      PrivateTmp = true;
+    };
+  };
+in {
+  users.groups.mc-wow-tunnel = { };
+  users.users.mc-wow-tunnel = {
+    isSystemUser = true;
+    group = "mc-wow-tunnel";
+  };
+  sops.secrets."minecraft/wow-tunnel-key" = {
+    sopsFile = ../../../../sus/neuro-minecraft.yaml;
+    owner = "mc-wow-tunnel";
+    group = "mc-wow-tunnel";
+    mode = "0400";
+  };
+  programs.ssh.knownHosts.minecraft-wow-relay = {
+    hostNames = [ "128.140.75.58" ];
+    publicKey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIAFpr4DPSaJt0xeuGIfcZBJD3LsJHTdIRIs2Tt9HF+CT";
+  };
+  programs.ssh.knownHosts.minecraft-wow-relay-bfs = {
+    hostNames = [ "91.198.166.181" ];
+    publicKey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAICcCn57nlWY5QyEz17kxuAbIX9PkjPwtlGzdJyhy+SQQ";
+  };
+  systemd.services.minecraft-wow-tunnel = mkTunnel {
+    name = "hectic-lab";
+    address = "128.140.75.58";
+  };
+  systemd.services.minecraft-wow-tunnel-bfs = mkTunnel {
+    name = "bfs.band";
+    address = "91.198.166.181";
+  };
+}
@@ -0,0 +1,18 @@
+{ pkgs, ... }:
+
+{
+  services.minecraft-servers.servers.vanilla = {
+    enable = true;
+    jvmOpts = "-Xmx6G -Xms2G";
+    package = pkgs.minecraftServers.vanilla-1_21_11;
+
+    serverProperties = {
+      server-port = 25565;
+      difficulty = "hard";
+      online-mode = true;
+      view-distance = 32;
+      level-seed = "8306359138650378643";
+      pause-when-empty-seconds = 0;
+    };
+  };
+}
@@ -0,0 +1,312 @@
+{
+  config,
+  lib,
+  pkgs,
+  ...
+}: let
+  inherit (lib) mkOption types;
+
+  cfg = config.services.minecraft-world-imports;
+  enabledImports = lib.filterAttrs (_: import: import.enable) cfg;
+  dataDir = config.services.minecraft-servers.dataDir;
+  minecraftServers = config.services.minecraft-servers.servers;
+  targetPairs = lib.mapAttrsToList (_: import:
+    "${import.serverName}:${import.worldName}") enabledImports;
+
+  importerUser = name: let
+    descriptiveName = "minecraft-map-import-${name}";
+  in
+    if builtins.stringLength descriptiveName <= 31
+    then descriptiveName
+    else "mc-import-${builtins.substring 0 12 (builtins.hashString "sha256" name)}";
+
+  stateDirectory = cacheDir:
+    if lib.hasPrefix "/var/lib/" cacheDir
+    then lib.removePrefix "/var/lib/" cacheDir
+    else null;
+
+  stateDirectoryCompatible = cacheDir: let
+    relative = stateDirectory cacheDir;
+    components = lib.splitString "/" (if relative == null then "" else relative);
+  in
+    relative != null
+    && relative != ""
+    && lib.all (component: component != "" && component != "." && component != "..") components;
+
+  escapeSftp = value:
+    "\"${lib.replaceStrings ["\\" "\""] ["\\\\" "\\\""] value}\"";
+
+  escapeTmpfiles = value:
+    lib.replaceStrings ["%" " " "\t"] ["%%" "\\x20" "\\x09"] value;
+
+  importerServices = lib.mkMerge (lib.mapAttrsToList (name: import: let
+    user = importerUser name;
+    unitName = "minecraft-world-import-${name}";
+    serverUnit = "minecraft-server-${import.serverName}.service";
+    cacheStateDirectory = stateDirectory import.cacheDir;
+    serverDir = "${dataDir}/${import.serverName}";
+    worldDir = "${serverDir}/${import.worldName}";
+  in {
+    ${unitName} = {
+      description = "Import Minecraft world ${name}";
+      before = [ serverUnit ];
+      wants = [ "network-online.target" ];
+      after = [ "network-online.target" ];
+      path = [ pkgs.coreutils pkgs.findutils pkgs.openssh pkgs.unar ];
+      serviceConfig = {
+        Type = "oneshot";
+        User = user;
+        Group = user;
+        RemainAfterExit = true;
+        TimeoutStartSec = import.timeoutStartSec;
+        ProtectSystem = "strict";
+        ProtectHome = true;
+        PrivateDevices = true;
+        PrivateTmp = true;
+        NoNewPrivileges = true;
+        CapabilityBoundingSet = [ "" ];
+        ReadWritePaths = [ import.cacheDir dataDir ];
+        UMask = "0077";
+      } // lib.optionalAttrs (stateDirectoryCompatible import.cacheDir) {
+        StateDirectory = cacheStateDirectory;
+      };
+      script = ''
+        set -eu
+        umask 077
+
+        cache_dir=${lib.escapeShellArg import.cacheDir}
+        server_dir=${lib.escapeShellArg serverDir}
+        world_dir=${lib.escapeShellArg worldDir}
+        archive_name=${lib.escapeShellArg import.archiveName}
+        world_name=${lib.escapeShellArg import.worldName}
+        archive="$cache_dir/$archive_name"
+        temporary_archive="$cache_dir/.$archive_name.$$"
+        extraction_dir="$cache_dir/.minecraft-world-import-${name}.$$"
+        staged_world="$server_dir/.$world_name.import.$$"
+        key=${lib.escapeShellArg import.sshKeyFile}
+
+        cleanup() {
+          rm -f "$temporary_archive"
+          rm -rf "$extraction_dir" "$staged_world"
+        }
+        trap cleanup EXIT
+
+        mkdir -p "$cache_dir" "$server_dir"
+        chmod 0700 "$cache_dir"
+        # The Minecraft module creates server_dir as minecraft:minecraft
+        # with group write access; this importer must not chmod another user's directory.
+
+        if [ -d "$world_dir" ]; then
+          if [ -f "$world_dir/level.dat" ]; then
+            exit 0
+          fi
+          echo "Minecraft world directory exists but has no level.dat" >&2
+          exit 1
+        fi
+
+        if [ ! -f "$archive" ]; then
+          downloaded=false
+          attempt=1
+          while [ "$attempt" -le ${toString import.downloadRetries} ]; do
+            if sftp \
+              -o BatchMode=yes \
+              -o StrictHostKeyChecking=yes \
+              -o UserKnownHostsFile=/etc/ssh/ssh_known_hosts \
+              -i "$key" \
+              -b - \
+              ${lib.escapeShellArg "${import.remoteUser}@${import.remoteHost}"} <<EOF
+        get ${escapeSftp import.remotePath} "$temporary_archive"
+        EOF
+            then
+              downloaded=true
+              break
+            fi
+            rm -f "$temporary_archive"
+            sleep ${toString import.retryDelaySeconds}
+            attempt=$((attempt + 1))
+          done
+          if [ "$downloaded" != true ]; then
+            echo "Unable to download Minecraft world ${name}" >&2
+            exit 1
+          fi
+          mv "$temporary_archive" "$archive"
+        fi
+
+        if ! printf '%s  %s\n' \
+            ${lib.escapeShellArg import.archiveSha256} \
+            "$archive" | sha256sum -c -; then
+          rm -f "$archive"
+          echo "Cached Minecraft world ${name} checksum mismatch" >&2
+          exit 1
+        fi
+
+        mkdir -p "$extraction_dir"
+        unar -quiet -output-directory "$extraction_dir" "$archive"
+
+        find "$extraction_dir" \
+          \( -type l -o -type b -o -type c -o -type p -o -type s \) \
+          -delete
+
+        world_level_dat=$(find "$extraction_dir" -type f -name level.dat -print -quit)
+        if [ -z "$world_level_dat" ]; then
+          echo "Minecraft world archive contains no level.dat" >&2
+          exit 1
+        fi
+
+        mv "$(dirname "$world_level_dat")" "$staged_world"
+        chgrp -R minecraft "$staged_world"
+        chmod -R u+rwX,g+rwX,o-rwx "$staged_world"
+        mv "$staged_world" "$world_dir"
+      '';
+    };
+
+    "minecraft-server-${import.serverName}" = {
+      requires = [ "${unitName}.service" ];
+      after = [ "${unitName}.service" ];
+    };
+  }) enabledImports);
+in {
+  options.services.minecraft-world-imports = mkOption {
+    default = { };
+    type = types.attrsOf (types.submodule ({ name, ... }: {
+      options = {
+        enable = lib.mkEnableOption "Minecraft world import ${name}";
+
+        serverName = mkOption {
+          type = types.str;
+          description = "minecraft-servers server receiving imported world";
+        };
+
+        remoteHost = mkOption {
+          type = types.str;
+          description = "SSH host serving world archive";
+        };
+
+        remoteUser = mkOption {
+          type = types.str;
+          description = "SSH user used to download world archive";
+        };
+
+        remotePath = mkOption {
+          type = types.str;
+          description = "Remote path to world archive";
+        };
+
+        archiveName = mkOption {
+          type = types.str;
+          description = "Archive file name inside cache directory";
+        };
+
+        cacheDir = mkOption {
+          type = types.str;
+          default = "/var/lib/minecraft-world-imports/${name}";
+          description = "Persistent archive cache directory";
+        };
+
+        archiveSha256 = mkOption {
+          type = types.strMatching "[0-9a-fA-F]{64}";
+          description = "Expected SHA-256 digest of world archive";
+        };
+
+        sshKeyFile = mkOption {
+          type = types.str;
+          description = "Runtime path to private SSH key";
+        };
+
+        worldName = mkOption {
+          type = types.str;
+          default = "world";
+          description = "World directory name beneath server directory";
+        };
+
+        hostPublicKey = mkOption {
+          type = types.str;
+          description = "Pinned SSH host public key";
+        };
+
+        downloadRetries = mkOption {
+          type = types.ints.positive;
+          default = 3;
+          description = "Maximum SFTP download attempts";
+        };
+
+        retryDelaySeconds = mkOption {
+          type = types.ints.unsigned;
+          default = 10;
+          description = "Delay between SFTP download attempts";
+        };
+
+        timeoutStartSec = mkOption {
+          type = types.str;
+          default = "30min";
+          description = "Importer service start timeout";
+        };
+      };
+    }));
+    description = "Minecraft worlds imported before selected servers start";
+  };
+
+  config = lib.mkIf (enabledImports != { }) {
+    assertions = lib.flatten (lib.mapAttrsToList (name: import: [
+      {
+        assertion = builtins.match "[A-Za-z0-9_-]+" name != null;
+        message = "services.minecraft-world-imports.${name}: name must contain only letters, digits, underscores, or hyphens";
+      }
+      {
+        assertion = builtins.stringLength name <= 24;
+        message = "services.minecraft-world-imports.${name}: name must be at most 24 characters";
+      }
+      {
+        assertion = builtins.match "/.*" import.cacheDir != null;
+        message = "services.minecraft-world-imports.${name}.cacheDir must be absolute";
+      }
+      {
+        assertion = builtins.match "[A-Za-z0-9_-]+" import.serverName != null;
+        message = "services.minecraft-world-imports.${name}.serverName must contain only letters, digits, underscores, or hyphens";
+      }
+      {
+        assertion = !lib.hasInfix "\n" import.remotePath && !lib.hasInfix "\r" import.remotePath;
+        message = "services.minecraft-world-imports.${name}.remotePath must not contain newlines";
+      }
+      {
+        assertion = builtins.hasAttr import.serverName minecraftServers
+          && (builtins.getAttr import.serverName minecraftServers).enable;
+        message = "services.minecraft-world-imports.${name}.serverName must name an enabled Minecraft server";
+      }
+      {
+        assertion = lib.length (lib.unique targetPairs) == lib.length targetPairs;
+        message = "services.minecraft-world-imports: each server/world target must be unique";
+      }
+      {
+        assertion = builtins.match "[^/]+" import.archiveName != null;
+        message = "services.minecraft-world-imports.${name}.archiveName must be a file name";
+      }
+      {
+        assertion = builtins.match "[^/]+" import.worldName != null;
+        message = "services.minecraft-world-imports.${name}.worldName must be a directory name";
+      }
+    ]) enabledImports);
+
+    users.groups = lib.mapAttrs' (name: _: lib.nameValuePair (importerUser name) { }) enabledImports;
+    users.users = lib.mapAttrs' (name: _: let
+      user = importerUser name;
+    in lib.nameValuePair user {
+      description = "Minecraft world importer ${name}";
+      isSystemUser = true;
+      group = user;
+      extraGroups = [ "minecraft" ];
+    }) enabledImports;
+
+    programs.ssh.knownHosts = lib.mapAttrs' (name: import:
+      lib.nameValuePair "minecraft-world-import-${name}" {
+        hostNames = [ import.remoteHost ];
+        publicKey = import.hostPublicKey;
+      }) enabledImports;
+
+    systemd.tmpfiles.rules = lib.flatten (lib.mapAttrsToList (name: import:
+      lib.optional (!stateDirectoryCompatible import.cacheDir)
+        "d ${escapeTmpfiles import.cacheDir} 0700 ${importerUser name} ${importerUser name} -") enabledImports);
+
+    systemd.services = importerServices;
+  };
+}
@@ -0,0 +1,33 @@
+{ config, ... }:
+
+{
+  sops.secrets."minecraft/storage-box-pack-key" = {
+    sopsFile = ../../../../sus/neuro-minecraft.yaml;
+    owner = "mc-pack-worldOfSosal";
+    group = "mc-pack-worldOfSosal";
+    mode = "0400";
+  };
+
+  services.minecraft-modpack-imports.worldOfSosal = {
+    enable = true;
+    serverName = "wowMineMap";
+    remoteHost = "u664722.your-storagebox.de";
+    remoteUser = "u664722";
+    remotePath = "minecraft/pack/WorldOfSosal-v3.mrpack";
+    archiveName = "WorldOfSosal.mrpack";
+    cacheDir = "/var/lib/minecraft-modpacks/worldOfSosal";
+    archiveSha256 = "f97cf251b14f40590e97e7b39e8a8ec43dacfce6da1b02357d15e0eee10d3ade";
+    expectedDependencies = {
+      minecraft = "1.21.1";
+      neoforge = "21.1.250";
+    };
+    sshKeyFile = config.sops.secrets."minecraft/storage-box-pack-key".path;
+    hostPublicKey = "ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEA5EB5p/5Hp3hGW1oHok+PIOH9Pbn7cnUiGmUEBrCVjnAw+HrKyN8bYVV0dIGllswYXwkG/+bgiBlE6IVIBAq+JwVWu1Sss3KarHY3OvFJUXZoZyRRg/Gc/+LRCE7lyKpwWQ70dbelGRyyJFH36eNv6ySXoUYtGkwlU5IVaHPApOxe4LHPZa/qhSRbPo2hwoh0orCtgejRebNtW5nlx00DNFgsvn8Svz2cIYLxsPVzKgUxs8Zxsxgn+Q/UvR7uq4AbAhyBMLxv7DjJ1pc7PJocuTno2Rw9uMZi1gkjbnmiOh6TTXIEWbnroyIhwc8555uto9melEUmWNQ+C+PwAK+MPw==";
+  };
+
+  # Import the map before writing modpack configuration into the same server.
+  systemd.services.minecraft-modpack-import-worldOfSosal = {
+    after = [ "minecraft-world-import-wowMineMap.service" ];
+    requires = [ "minecraft-world-import-wowMineMap.service" ];
+  };
+}
@@ -0,0 +1,65 @@
+{ config, pkgs, ... }:
+
+{
+  sops.secrets."minecraft/storage-box-key" = {
+    sopsFile = ../../../../sus/neuro-minecraft.yaml;
+    owner = "minecraft-map-import-wowMineMap";
+    group = "minecraft-map-import-wowMineMap";
+    mode = "0400";
+  };
+
+  services.minecraft-world-imports.wowMineMap = {
+    enable = true;
+    serverName = "wowMineMap";
+    remoteHost = "u664722.your-storagebox.de";
+    remoteUser = "u664722";
+    remotePath = "minecraft/map/wow mine map.rar";
+    archiveName = "wow mine map.rar";
+    cacheDir = "/var/lib/minecraft-maps";
+    archiveSha256 = "bc80084de10a06b0fc2cb1651c61936b9e2fd2288f3f0fe44c964d83a393aa30";
+    sshKeyFile = config.sops.secrets."minecraft/storage-box-key".path;
+    worldName = "world";
+    hostPublicKey = "ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEA5EB5p/5Hp3hGW1oHok+PIOH9Pbn7cnUiGmUEBrCVjnAw+HrKyN8bYVV0dIGllswYXwkG/+bgiBlE6IVIBAq+JwVWu1Sss3KarHY3OvFJUXZoZyRRg/Gc/+LRCE7lyKpwWQ70dbelGRyyJFH36eNv6ySXoUYtGkwlU5IVaHPApOxe4LHPZa/qhSRbPo2hwoh0orCtgejRebNtW5nlx00DNFgsvn8Svz2cIYLxsPVzKgUxs8Zxsxgn+Q/UvR7uq4AbAhyBMLxv7DjJ1pc7PJocuTno2Rw9uMZi1gkjbnmiOh6TTXIEWbnroyIhwc8555uto9melEUmWNQ+C+PwAK+MPw==";
+  };
+
+  sops.secrets."minecraft/rcon-password" = {
+    sopsFile = ../../../../sus/neuro-minecraft.yaml;
+    owner = "minecraft";
+    group = "minecraft";
+    mode = "0400";
+    restartUnits = [ "minecraft-server-wowMineMap.service" ];
+  };
+
+  # The module's automatic firewall would also expose RCON.
+  networking.firewall.allowedTCPPorts = [ 25567 ];
+  services.minecraft-servers.servers.wowMineMap = {
+    openFirewall = false;
+    extraStartPre = ''
+      chmod 600 server.properties
+      {
+        printf '\nrcon.password='
+        cat ${config.sops.secrets."minecraft/rcon-password".path}
+        printf '\n'
+      } >> server.properties
+    '';
+    enable = true;
+    jvmOpts = "-Xmx24G -Xms2G";
+    # WorldOfSosal client and server use the same pinned NeoForge.
+    package = pkgs.minecraftServers.neoforge-1_21_1.override (
+      builtins.fromJSON (builtins.readFile ./neoforge-21.1.250.json)
+    );
+
+    serverProperties = {
+      server-port = 25567;
+      difficulty = "hard";
+      online-mode = false;
+      enable-rcon = true;
+      "rcon.port" = 25575;
+      view-distance = 12;
+      simulation-distance = 8;
+      motd = "WorldOfSosal — World of Warcraft";
+      level-name = "world";
+      pause-when-empty-seconds = 0;
+    };
+  };
+}
@@ -17,13 +17,18 @@

  ollamaServiceBundledLibraryPath = "${ollamaPrebuilt}/lib/ollama:${ollamaPrebuilt}/lib/ollama/cuda_v12:${ollamaPrebuilt}/lib/ollama/cuda_v13";

+  stableVideoDiffusionLibraryPath = lib.makeLibraryPath [
+    pkgs.stdenv.cc.cc.lib
+    pkgs.zlib
+  ];
+
  ollamaPrebuilt = pkgs.stdenvNoCC.mkDerivation {
    pname = "ollama";
-    version = "0.22.1";
+     version = "0.24.0";

    src = pkgs.fetchurl {
-      url = "https://github.com/ollama/ollama/releases/download/v0.22.1/ollama-linux-amd64.tar.zst";
-      hash = "sha256-4nwP6PYKgkFi+Bzge0v9p2fc5PNX12LhSbPQ3gq62fs=";
+       url = "https://github.com/ollama/ollama/releases/download/v0.24.0/ollama-linux-amd64.tar.zst";
+       hash = "sha256-FcX41mugbg07RxnfiGhhLb1m4U6CdgkpuzVS4WV83Ns=";
    };

    nativeBuildInputs = [
@@ -65,7 +70,7 @@ in {
  imports = [
    self.nixosModules.hectic
    inputs.sops-nix.nixosModules.sops
-    ./minecraft.nix
+    ./minecraft
    ./hardware.nix 
  ];

@@ -134,7 +139,7 @@ in {
  };

  services.nginx = {
-    enable = true;
+    enable = false;
    virtualHosts."bfs.band" = let
      site = pkgs.runCommand "bfs-band-site" {} ''
        mkdir -p $out
@@ -176,6 +181,26 @@ in {
    openFirewall = false;
  };

+  hectic.services.stable-video-diffusion = {
+    enable       = true;
+    host         = "127.0.0.1";
+    port         = 7861;
+    package      = pkgs.hectic.stable-video-diffusion-api;
+    device       = "cuda";
+    libraryPath  = [
+      stableVideoDiffusionLibraryPath
+      "/run/opengl-driver/lib"
+    ];
+    openFirewall = false;
+  };
+
+  hectic.services.dify = {
+    enable = true;
+    environmentFile = config.sops.secrets."dify/environment".path;
+    port = 8080;
+    pluginPort = 5003;
+  };
+
  networking = {
    networkmanager.enable = true;
    useDHCP = lib.mkDefault true;
@@ -199,6 +224,70 @@ in {
    archetype.dev.enable  = true;
  };

+  hectic.services."project-zomboid" = {
+    enable = true;
+    memory = "8g";
+    serverName = "servertest";
+    serverPropertiesFile = /var/lib/project-zomboid/server-password.ini;
+    serverProperties = {
+      Map = "Muldraugh, KY";
+      DoLuaChecksum = false;
+      Public = true;
+      AntiCheatSafety = 4;
+      AntiCheatMovement = 4;
+      AntiCheatSpeed = 4;
+      AntiCheatHit = 4;
+      AntiCheatPacket = 4;
+      AntiCheatPacketException = 4;
+      AntiCheatPermission = 4;
+      AntiCheatXP = 4;
+      AntiCheatFire = 4;
+      AntiCheatSafeHouse = 4;
+      AntiCheatRecipe = 4;
+      AntiCheatPlayer = 4;
+      AntiCheatChecksum = 4;
+      AntiCheatItem = 4;
+      AntiCheatNoClip = 4;
+      AntiCheatServerCustomization = 4;
+    };
+    workshopItems = [
+      "3676456221" # Lua Digital Watch Framework
+      "3600401184" # Realistic Temperature Mod
+    ];
+    mods = [
+      "\\LuaDigitalWatchUI"
+      "\\RC_RealisticColdMod"
+    ];
+    sandboxProperties = {
+      Zombies = 6;
+      ZombieConfig = {
+        PopulationMultiplier = 0.0;
+        PopulationStartMultiplier = 0.0;
+        PopulationPeakMultiplier = 0.0;
+        RespawnHours = 0.0;
+        RespawnUnseenHours = 0.0;
+        RespawnMultiplier = 0.0;
+        RedistributeHours = 0.0;
+      };
+    };
+  };
+
+  systemd.services.project-zomboid.preStart = lib.mkBefore ''
+    password_file=${lib.escapeShellArg "/var/lib/project-zomboid/server-password"}
+    properties_file=${lib.escapeShellArg "/var/lib/project-zomboid/server-password.ini"}
+
+    if [ ! -s "$password_file" ] || ! ${pkgs.gnugrep}/bin/grep -Eq '^[0-9a-f]{48}$' "$password_file"; then
+      umask 077
+      ${pkgs.openssl}/bin/openssl rand -hex 24 > "$password_file"
+    fi
+    ${pkgs.coreutils}/bin/chmod 0600 "$password_file"
+
+    properties_file_tmp="$( ${pkgs.coreutils}/bin/mktemp "$(dirname "$properties_file")/.server-password.ini.XXXXXX")"
+    ${pkgs.coreutils}/bin/printf 'Password=%s\n' "$(<"$password_file")" > "$properties_file_tmp"
+    ${pkgs.coreutils}/bin/chmod 0600 "$properties_file_tmp"
+    ${pkgs.coreutils}/bin/mv "$properties_file_tmp" "$properties_file"
+  '';
+
  sops = {
    gnupg.sshKeyPaths         = [ ];
    age.sshKeyPaths           = [ "/etc/ssh/ssh_host_ed25519_key" ];
@@ -211,6 +300,7 @@ in {
    #  group = "turnserver";
    #  mode = "0400";
    #};
+    secrets."dify/environment" = {};
  };

  boot.loader.systemd-boot.enable = true;
@@ -263,6 +353,7 @@ in {
  in [
    #python-ai
    git
+    hectic.hiddify-core
    neovim
    wget
    ethtool
@@ -0,0 +1,20 @@
+{
+  flake,
+  self,
+  inputs,
+  system ? "x86_64-linux",
+  ...
+}: let
+  # Use folder name as name of this system
+  name = builtins.baseNameOf ./.;
+
+in self.lib.nixpkgs-lib.nixosSystem {
+  pkgs = import inputs.nixpkgs {
+    inherit system;
+    overlays = [ self.overlays.default ];
+  };
+  modules = [
+    { networking.hostName = name; }
+    (import ./${name}.nix { inherit flake self inputs; })
+  ];
+}
@@ -0,0 +1,35 @@
+{
+  inputs,
+  self,
+  ...
+}: {
+  pkgs,
+  modulesPath,
+  ...
+}: {
+  imports = [
+    (modulesPath + "/profiles/qemu-guest.nix")
+    inputs.sops-nix.nixosModules.sops
+    self.nixosModules.hectic
+  ];
+
+  hectic = {
+    archetype.dev.enable = true;
+    hardware.hetzner-cloud = {
+      enable                = true;
+      device                = "/dev/sda";
+      networkMatchConfigMac = "92:00:08:4a:b0:32";
+      ipv4                  = "46.225.237.218";
+      ipv6                  = "2a01:4f8:c2c:3b14";
+    };
+  };
+
+  users.users.root.openssh.authorizedKeys.keys = [
+    ''ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIKAaObjLBslsdTlqEcYaS1TqX4x9aVJu75y27/8MFevO''
+  ];
+
+  environment.systemPackages = with pkgs; [
+    git
+    rsync
+  ];
+}
@@ -1,18 +1,21 @@
 # db-tool

-PostgreSQL development database management tool. Drop-in replacement for per-project database.sh / postgres-init.sh / postgres-cleanup.sh scripts. Provides database, postgres-init, and postgres-cleanup binaries.
+PostgreSQL utility package exposing separate development and operations binaries.
+`db-dev` preserves the current development workflow via the `database` binary;
+`db-ops` provides target-safe operational helpers such as secrets hydration.

 ## Provided Binaries

 | Binary | Description |
 | --- | --- |
-| `database` | Main script for managing migrations, deployments, and logs. |
+| `database` | Main `db-dev` script for managing local development databases, migrations, deployments, and logs. |
+| `db-ops` | Operational helper binary for target/runtime database actions. |
 | `postgres-init` | Ephemeral PostgreSQL cluster initialization and startup. |
 | `postgres-cleanup` | Graceful shutdown and cleanup of the PostgreSQL cluster. |

 ## Required Environment Variables

-These variables must be set for `db-tool` to function.
+These variables must be set for `db-dev` / `database` to function.

 | Variable | Description |
 | --- | --- |
@@ -39,7 +42,7 @@ These variables must be set for `db-tool` to function.

 ## Postgres Package Override

-By default, `db-tool`/`postgres-init`/`postgres-cleanup` use plain `postgresql_17` from nixpkgs. If you need extensions (e.g. `pg_cron`), override the postgres package per-output:
+By default, `db-dev`/`db-ops`/`postgres-init`/`postgres-cleanup` use plain `postgresql_17` from nixpkgs. If you need extensions (e.g. `pg_cron`), override the postgres package per-output:

 ```nix
 let
@@ -48,7 +51,8 @@ let
  ]);
 in {
  packages = [
-    (pkgs.hectic."db-tool".override          { postgresql = myPg; })
+    (pkgs.hectic."db-dev".override           { postgresql = myPg; })
+    (pkgs.hectic."db-ops".override           { postgresql = myPg; })
    (pkgs.hectic."postgres-init".override    { postgresql = myPg; })
    (pkgs.hectic."postgres-cleanup".override { postgresql = myPg; })
  ];
@@ -66,6 +70,28 @@ The `pull_staging` subcommand allows importing data from a remote staging enviro

 If any of these variables are missing when `pull_staging` is invoked, the tool will exit with code 3 and print the name of the missing variable to stderr.

+## normalize-backup Contract
+
+The `normalize-backup` subcommand converts a physical PostgreSQL backup into a
+local-development restore artifact without modifying the input backup:
+
+```sh
+database normalize-backup [OPTIONS] [path]
+```
+
+The input `path` defaults to `${LOCAL_DIR}/focus/postgresql-backup` and must be a
+backup directory compatible with `database restore`, containing `base.tar.gz`
+and optionally `pg_wal.tar.gz`. The output defaults to a normalized backup path
+and can be overridden with `--output <path>`. The output directory must not be
+the same path as the input and must not be inside the input directory.
+
+Normalization extracts the physical backup into temporary PGDATA, replaces
+production PostgreSQL access/configuration with local restore-safe
+`postgresql.conf` and `pg_hba.conf` files, removes standby/recovery/runtime
+leftovers, starts the cluster locally, ensures the requested `--role` and
+`--database` exist, stops cleanly, and repacks a backup directory that can be
+used by `database restore`.
+
 ## Subcommands

 - `deploy`: Execute the full deployment flow (hydrate + patch). Supports `--cleanup` to teardown after success.
@@ -73,20 +99,21 @@ If any of these variables are missing when `pull_staging` is invoked, the tool w
 - `test`: Execute database tests located in `${DATABASE_DIR}/test/test.sql`.
 - `check`: Run a deployment validation in an isolated, temporary PostgreSQL cluster.
 - `cleanup`: Stop the local database cluster and remove the `PG_WORKING_DIR`.
+- `normalize-backup`: Rewrite a physical backup artifact for local restore use.
 - `pull_staging`: Import data from the staging environment based on the env contract.
 - `init`: Wrapper around `postgres-init` to start the cluster.
 - `migrator`: Directly invoke the migration tool with the correct environment context.

 ## shellHook Example

-To use `db-tool` in a Nix development shell, add the following to your `flake.nix` or `shell.nix`:
+To use `db-dev` in a Nix development shell, add the following to your `flake.nix` or `shell.nix`:

 ```nix
 {
  # ...
  devShells.default = pkgs.mkShell {
    packages = [
-      pkgs.hectic.db-tool
+      pkgs.hectic.db-dev
      pkgs.hectic.postgres-init
      pkgs.hectic.postgres-cleanup
    ];
@@ -111,7 +138,7 @@ To use `db-tool` in a Nix development shell, add the following to your `flake.ni

 ## hectic Bundle

-`db-tool` and `migrator` apply a single bundle of SQL files that bootstrap the
+`db-dev`, `db-ops`, and `migrator` apply a single bundle of SQL files that bootstrap the
 `hectic` schema. The bundle lives in
 [`lib/hook/sql/`](../../lib/hook/sql/README.md) — see that README for full
 contract, file layout, and the `self.lib.hectic.*` Nix API.
@@ -167,6 +194,7 @@ ALTER DATABASE mydb SET hectic.inheritance_extra_excluded_schemas = 'legacy,etl'
 | `postgres-init` | **No.** Pure PostgreSQL provisioner — starts a vanilla cluster, nothing more. |
 | `migrator init` | **Yes, mandatory.** The bundle is a hard prerequisite for `hectic.migration`. |
 | `database hydrate` | **Yes, by default.** Re-applied on every hydrate. Skip with `--no-hook`. After applying the bundle, hydrate also calls `hectic.load_secrets_from_env(<dotenv>)` if `HECTIC_DOTENV_FILE` (or `${LOCAL_DIR}/.env.${ENVIRONMENT}`) is readable. |
+| `db-ops secrets load` | **Yes, mandatory.** Applies the bundle and requires a dotenv source before loading secrets into `hectic.secret`. |

 The bundle is idempotent — repeated application is safe.

@@ -187,15 +215,13 @@ directly against the paths exposed by `self.lib.hectic.*.path`:

 ```nix
 services.postgresql.initialScript = pkgs.writeText "hectic-init.sql" ''
+  \i ${self.lib.hectic.version.path}
  \i ${self.lib.hectic.secret.path}
  \i ${self.lib.hectic.migration.path}
  \i ${self.lib.hectic.inheritance.path}
 '';
 ```

-The version file (`self.lib.hectic.version`) is templated and only exposes
-`.sql` (a string). Materialize it with `pkgs.writeText` if a path is needed.
-
 ## Exit Codes

 | Code | Meaning |
@@ -4,6 +4,7 @@
 : "${REMAINING_ARGS:=}"

 : "${DEFAULT_BACKUP_PATH:=${LOCAL_DIR:-$PWD}/focus/postgresql-backup}"
+: "${DEFAULT_NORMALIZED_BACKUP_PATH:=${LOCAL_DIR:-$PWD}/focus/postgresql-backup-normalized}"

 : "${SCRIPT_NAME:=$(basename "$0")}"
 SCRIPT_NAME=${SCRIPT_NAME%%.sh}
@@ -186,6 +187,9 @@ ${BGREEN}Database Subcommands:${NC}

  ${BCYAN}restore${NC} ${CYAN}[PATH]$NC           Restore database from backup

+  ${BCYAN}normalize-backup${NC} ${CYAN}[OPTIONS] [PATH]$NC
+                            Normalize a raw physical backup for local restore/diff
+
  ${BCYAN}backup${NC}                   Create database backup
                           Creates compressed backup at $BBLACK$DEFAULT_BACKUP_PATH$NC

@@ -246,6 +250,7 @@ ${BGREEN}Examples:${NC}
  $SCRIPT_NAME init                       Initialize PostgreSQL only
  $SCRIPT_NAME restore                    Restore from default backup
  $SCRIPT_NAME restore /path/to/backup    Restore from specific path
+  $SCRIPT_NAME normalize-backup /path/to/raw-backup
  $SCRIPT_NAME backup                     Create backup
  $SCRIPT_NAME log                        Show database logs
  $SCRIPT_NAME log list                   List available log files
@@ -421,6 +426,43 @@ EOF
 )" | "$PAGER_OR_CAT"
 }

+help_normalize_backup() {
+  # shellcheck disable=SC2059
+  printf "$(cat <<EOF
+${BGREEN}Usage:${NC} $SCRIPT_NAME normalize-backup [OPTIONS] [path]
+
+Normalize a raw PostgreSQL physical backup for local development.
+
+This command leaves the input backup untouched. It extracts $BBLACK\`base.tar.gz\`$NC
+and optional $BBLACK\`pg_wal.tar.gz\`$NC into temporary PGDATA, writes local
+$BBLACK\`postgresql.conf\`$NC and $BBLACK\`pg_hba.conf\`$NC files, removes standby,
+recovery, and runtime leftovers, starts the cluster locally, ensures a login role
+and database exist, stops cleanly, then repacks a normalized backup directory.
+
+${BGREEN}Arguments:${NC}
+  ${BCYAN}path$NC                  Input backup directory (optional)
+                        Defaults to $BBLACK$DEFAULT_BACKUP_PATH$NC
+
+${BGREEN}Options:${NC}
+  $BCYAN-o$NC, $BCYAN--output$NC ${CYAN}<path>$NC      Output backup directory
+                        Defaults to $BBLACK$DEFAULT_NORMALIZED_BACKUP_PATH$NC
+  $BCYAN--role$NC ${CYAN}<name>$NC          Login role to ensure (default $BBLACK\$(id -un)$NC)
+  $BCYAN--database$NC ${CYAN}<name>$NC      Database to ensure (default $BBLACK\${PG_DATABASE:-testdb}$NC)
+  $BCYAN--admin-role$NC ${CYAN}<name>$NC    Role used for local maintenance
+                        Defaults to $BBLACK\${URI_USER:-postgres}$NC
+  $BCYAN-h$NC, $BCYAN--help$NC            Show this help message
+
+${BGREEN}Files Created:${NC}
+  - ${BBLACK}\`base.tar.gz\`$NC:      Normalized database cluster backup
+
+${BGREEN}Examples:${NC}
+  $SCRIPT_NAME normalize-backup /path/to/raw-backup
+  $SCRIPT_NAME normalize-backup --output focus/postgresql-backup-normalized
+
+EOF
+)" | "$PAGER_OR_CAT"
+}
+
 help_diff() {
  # shellcheck disable=SC2059
  printf "$(cat <<EOF
@@ -441,6 +483,8 @@ ${BGREEN}Arguments:
 ${BGREEN}Options:${NC}
  $BCYAN--tables${NC} ${CYAN}<list>${NC}     Comma-separated list of tables to diff
                      Example: --tables users,orders,products
+  $BCYAN--ignore-migration-fail${NC}
+                      Continue diff even if DB1 migrations fail
  $BCYAN-m${NC}, $BCYAN--mock${NC}          Mock external API calls when hydrating DB2
                      Replaces HTTP-calling functions with stubs/test data
  $BCYAN-h${NC}, $BCYAN--help${NC}          Show this help message
@@ -464,6 +508,7 @@ ${BGREEN}Examples:${NC}
  $SCRIPT_NAME diff                              Compare using default backup
  $SCRIPT_NAME diff /path/to/backup              Compare using specific backup
  $SCRIPT_NAME diff --tables users,orders        Compare specific tables
+  $SCRIPT_NAME diff --ignore-migration-fail      Continue despite DB1 migration failure
  $SCRIPT_NAME diff -m                           Compare with external APIs mocked
  $SCRIPT_NAME diff log                          Show logs from diff operation

@@ -805,6 +850,11 @@ subcommand_restore() {
    RESTORE_BACKUP_PATH="$DEFAULT_BACKUP_PATH"
  fi

+  local restore_database="${PG_DATABASE:-}"
+  if [ -f "${RESTORE_BACKUP_PATH:?}/database_name" ]; then
+    restore_database=$(tr -d '\n' < "${RESTORE_BACKUP_PATH:?}/database_name")
+  fi
+
  postgres-cleanup

  local data="${PG_WORKING_DIR:?}/data"
@@ -817,12 +867,243 @@ subcommand_restore() {
    tar -xzf "${RESTORE_BACKUP_PATH:?}/pg_wal.tar.gz" -C "${data}/pg_wal"
  fi

-  env PG_REUSE= postgres-init
+  env PG_REUSE= PG_DATABASE="$restore_database" postgres-init

  rm -f "${data}/standby.signal" "${data}/recovery.signal"
  restore_namespace
 }

+___sql_literal() {
+  printf "'%s'" "$(printf '%s' "$1" | sed "s/'/''/g")"
+}
+
+___normalize_backup_remove_leftovers() {
+  local pgdata="$1"
+  rm -f \
+    "$pgdata/postmaster.pid" \
+    "$pgdata/postmaster.opts" \
+    "$pgdata/recovery.signal" \
+    "$pgdata/standby.signal" \
+    "$pgdata/backup_label.old"
+  rm -f "$pgdata/pg_wal/archive_status"/*.ready 2>/dev/null || :
+}
+
+___normalize_backup_cleanup() {
+  local tmpdir="$1"
+  local pgdata="$2"
+  if [ -n "$pgdata" ] && [ -d "$pgdata" ]; then
+    pg_ctl -D "$pgdata" -m fast -w stop >/dev/null 2>&1 || :
+  fi
+  if [ -n "$tmpdir" ]; then
+    rm -rf "$tmpdir"
+  fi
+}
+
+___normalize_backup_refresh_collation() {
+  local sockdir="$1"
+  local port="$2"
+  local admin_role="$3"
+  local database_name="$4"
+
+  psql -h "$sockdir" -p "$port" -U "$admin_role" -d postgres \
+    -v ON_ERROR_STOP=1 -c "ALTER DATABASE \"$database_name\" REFRESH COLLATION VERSION;" \
+    >/dev/null 2>&1 || :
+}
+
+subcommand_normalize_backup() {
+  change_namespace 'db normalize-backup'
+
+  NORMALIZE_INPUT_PATH=""
+  NORMALIZE_OUTPUT_PATH="$DEFAULT_NORMALIZED_BACKUP_PATH"
+  NORMALIZE_ROLE="$(id -un)"
+  NORMALIZE_DATABASE="${PG_DATABASE:-testdb}"
+  NORMALIZE_ADMIN_ROLE="postgres"
+  NORMALIZE_PORT="${PG_PORT:-5432}"
+  NORMALIZE_PRELOAD_LIBRARIES="${PG_SHARED_PRELOAD_LIBRARIES:-pg_cron}"
+  NORMALIZE_DATABASE_EXPLICIT=0
+
+  while [ $# -gt 0 ]; do
+    case $1 in
+      -h|--help)
+        help_normalize_backup
+        exit 0
+      ;;
+      -o|--output)
+        if [ $# -lt 2 ]; then
+          log error "normalize-backup: --output requires a path"
+          exit 3
+        fi
+        NORMALIZE_OUTPUT_PATH="$2"
+        shift 2
+      ;;
+      --role)
+        if [ $# -lt 2 ]; then
+          log error "normalize-backup: --role requires a name"
+          exit 3
+        fi
+        NORMALIZE_ROLE="$2"
+        shift 2
+      ;;
+      --database)
+        if [ $# -lt 2 ]; then
+          log error "normalize-backup: --database requires a name"
+          exit 3
+        fi
+        NORMALIZE_DATABASE="$2"
+        NORMALIZE_DATABASE_EXPLICIT=1
+        shift 2
+      ;;
+      --admin-role)
+        if [ $# -lt 2 ]; then
+          log error "normalize-backup: --admin-role requires a name"
+          exit 3
+        fi
+        NORMALIZE_ADMIN_ROLE="$2"
+        shift 2
+      ;;
+      --*|-*)
+        log error "normalize-backup argument $1 does not exist"
+        exit 9
+      ;;
+      *)
+        if [ -n "$NORMALIZE_INPUT_PATH" ]; then
+          log error "normalize-backup: unexpected argument $1"
+          exit 9
+        fi
+        NORMALIZE_INPUT_PATH="$1"
+        shift
+      ;;
+    esac
+  done
+
+  if [ -z "$NORMALIZE_INPUT_PATH" ]; then
+    NORMALIZE_INPUT_PATH="$DEFAULT_BACKUP_PATH"
+  fi
+  if [ -z "$NORMALIZE_ROLE" ] || [ -z "$NORMALIZE_DATABASE" ] || [ -z "$NORMALIZE_ADMIN_ROLE" ]; then
+    log error "normalize-backup: role, database, and admin role must be non-empty"
+    exit 3
+  fi
+  if [ ! -d "$NORMALIZE_INPUT_PATH" ]; then
+    log error "normalize-backup: input backup directory not found: $NORMALIZE_INPUT_PATH"
+    exit 3
+  fi
+  if [ ! -f "$NORMALIZE_INPUT_PATH/base.tar.gz" ]; then
+    log error "normalize-backup: missing $NORMALIZE_INPUT_PATH/base.tar.gz"
+    exit 3
+  fi
+
+  NORMALIZE_INPUT_ABS=$(realpath "$NORMALIZE_INPUT_PATH")
+  NORMALIZE_OUTPUT_ABS=$(realpath -m "$NORMALIZE_OUTPUT_PATH")
+
+  if [ "$NORMALIZE_INPUT_ABS" = "$NORMALIZE_OUTPUT_ABS" ]; then
+    log error "normalize-backup: input and output paths must differ"
+    exit 1
+  fi
+  case "$NORMALIZE_OUTPUT_ABS/" in
+    "$NORMALIZE_INPUT_ABS"/*)
+      log error "normalize-backup: output path must not be inside input backup"
+      exit 1
+    ;;
+  esac
+  case "$NORMALIZE_INPUT_ABS/" in
+    "$NORMALIZE_OUTPUT_ABS"/*)
+      log error "normalize-backup: input backup must not be inside output path"
+      exit 1
+    ;;
+  esac
+  mkdir -p "$(dirname "$NORMALIZE_OUTPUT_ABS")"
+
+  NORMALIZE_TMPDIR=$(mktemp -d)
+  NORMALIZE_PGDATA="$NORMALIZE_TMPDIR/data"
+  NORMALIZE_SOCKDIR="$NORMALIZE_TMPDIR/sock"
+  NORMALIZE_LOG="$NORMALIZE_TMPDIR/postgres.log"
+  trap '___normalize_backup_cleanup "$NORMALIZE_TMPDIR" "$NORMALIZE_PGDATA"' EXIT INT HUP
+
+  mkdir -m 700 "$NORMALIZE_PGDATA"
+  mkdir -p "$NORMALIZE_SOCKDIR" "$NORMALIZE_PGDATA/pg_wal"
+
+  log notice "extracting backup into temporary PGDATA"
+  tar -xzf "$NORMALIZE_INPUT_ABS/base.tar.gz" -C "$NORMALIZE_PGDATA"
+  if [ -f "$NORMALIZE_INPUT_ABS/pg_wal.tar.gz" ]; then
+    tar -xzf "$NORMALIZE_INPUT_ABS/pg_wal.tar.gz" -C "$NORMALIZE_PGDATA/pg_wal"
+  fi
+
+  ___normalize_backup_remove_leftovers "$NORMALIZE_PGDATA"
+
+  if [ ! -f "$NORMALIZE_PGDATA/postgresql.conf" ]; then
+    cat > "$NORMALIZE_PGDATA/postgresql.conf" <<EOF
+listen_addresses = ''
+port = $NORMALIZE_PORT
+unix_socket_directories = '$NORMALIZE_SOCKDIR'
+logging_collector = off
+shared_preload_libraries = '$NORMALIZE_PRELOAD_LIBRARIES'
+cron.database_name = '$NORMALIZE_DATABASE'
+cron.host = '$NORMALIZE_SOCKDIR'
+EOF
+  fi
+  cat > "$NORMALIZE_PGDATA/pg_hba.conf" <<EOF
+local all all trust
+EOF
+  : > "$NORMALIZE_PGDATA/pg_ident.conf"
+  cat > "$NORMALIZE_PGDATA/postgresql.auto.conf" <<EOF
+# Local-dev normalized backup overrides.
+listen_addresses = ''
+port = '$NORMALIZE_PORT'
+unix_socket_directories = '$NORMALIZE_SOCKDIR'
+logging_collector = 'off'
+hba_file = '$NORMALIZE_PGDATA/pg_hba.conf'
+ident_file = '$NORMALIZE_PGDATA/pg_ident.conf'
+shared_preload_libraries = '$NORMALIZE_PRELOAD_LIBRARIES'
+cron.database_name = '$NORMALIZE_DATABASE'
+cron.host = '$NORMALIZE_SOCKDIR'
+EOF
+
+  log notice "starting temporary local cluster"
+  with_closed_fds pg_ctl -D "$NORMALIZE_PGDATA" -o "-F" -w start > "$NORMALIZE_LOG" 2>&1 || {
+    log error "normalize-backup: failed to start temporary cluster; see $NORMALIZE_LOG"
+    exit 1
+  }
+
+  ___normalize_backup_refresh_collation "$NORMALIZE_SOCKDIR" "$NORMALIZE_PORT" "$NORMALIZE_ADMIN_ROLE" postgres
+  ___normalize_backup_refresh_collation "$NORMALIZE_SOCKDIR" "$NORMALIZE_PORT" "$NORMALIZE_ADMIN_ROLE" template1
+  ___normalize_backup_refresh_collation "$NORMALIZE_SOCKDIR" "$NORMALIZE_PORT" "$NORMALIZE_ADMIN_ROLE" template0
+
+  if [ "$NORMALIZE_DATABASE_EXPLICIT" -eq 0 ]; then
+    detected_database=$(psql -h "$NORMALIZE_SOCKDIR" -p "$NORMALIZE_PORT" -U "$NORMALIZE_ADMIN_ROLE" -d postgres \
+      -tAc "SELECT datname FROM pg_database WHERE datallowconn AND NOT datistemplate AND datname <> 'postgres' ORDER BY oid LIMIT 1" 2>/dev/null | sed '/^$/d' | head -n 1)
+    if [ -n "$detected_database" ]; then
+      NORMALIZE_DATABASE="$detected_database"
+    fi
+  fi
+
+  NORMALIZE_ROLE_SQL=$(___sql_literal "$NORMALIZE_ROLE")
+  psql -h "$NORMALIZE_SOCKDIR" -p "$NORMALIZE_PORT" -U "$NORMALIZE_ADMIN_ROLE" -d postgres \
+    -v ON_ERROR_STOP=1 -c "DO \$\$ DECLARE v_role text := $NORMALIZE_ROLE_SQL; BEGIN IF NOT EXISTS (SELECT 1 FROM pg_roles WHERE rolname = v_role) THEN EXECUTE format('CREATE ROLE %I LOGIN SUPERUSER', v_role); ELSE EXECUTE format('ALTER ROLE %I LOGIN SUPERUSER', v_role); END IF; END \$\$;"
+
+  NORMALIZE_DATABASE_SQL=$(___sql_literal "$NORMALIZE_DATABASE")
+  NORMALIZE_DATABASE_EXISTS=$(psql -h "$NORMALIZE_SOCKDIR" -p "$NORMALIZE_PORT" -U "$NORMALIZE_ADMIN_ROLE" -d postgres \
+    -tAc "SELECT 1 FROM pg_database WHERE datname = $NORMALIZE_DATABASE_SQL" 2>/dev/null || true)
+  if [ "$NORMALIZE_DATABASE_EXISTS" != "1" ]; then
+    createdb -h "$NORMALIZE_SOCKDIR" -p "$NORMALIZE_PORT" -U "$NORMALIZE_ADMIN_ROLE" \
+      -O "$NORMALIZE_ROLE" "$NORMALIZE_DATABASE"
+  fi
+  ___normalize_backup_refresh_collation "$NORMALIZE_SOCKDIR" "$NORMALIZE_PORT" "$NORMALIZE_ADMIN_ROLE" "$NORMALIZE_DATABASE"
+
+  log notice "stopping temporary local cluster"
+  pg_ctl -D "$NORMALIZE_PGDATA" -m fast -w stop >> "$NORMALIZE_LOG" 2>&1
+  ___normalize_backup_remove_leftovers "$NORMALIZE_PGDATA"
+
+  log notice "writing normalized backup to $WHITE$NORMALIZE_OUTPUT_ABS$NC"
+  rm -rf "$NORMALIZE_OUTPUT_ABS"
+  mkdir -p "$NORMALIZE_OUTPUT_ABS"
+  tar -czf "$NORMALIZE_OUTPUT_ABS/base.tar.gz" -C "$NORMALIZE_PGDATA" .
+  printf '%s\n' "$NORMALIZE_DATABASE" > "$NORMALIZE_OUTPUT_ABS/database_name"
+
+  ___normalize_backup_cleanup "$NORMALIZE_TMPDIR" ""
+  trap - EXIT INT HUP
+  restore_namespace
+}
+
 subcommand_log() {
  change_namespace 'db log'
  : "${PG_WORKING_DIR:="$LOCAL_DIR/focus/postgresql"}"
@@ -1175,6 +1456,7 @@ ___diff_dump_schema() {
  local port="$2"
  local output_file="$3"
  local tables="${4:-}"
+  local database_name="${5:-${PG_DATABASE:-testdb}}"

  log info "dumping schema to $WHITE$output_file$NC"

@@ -1189,17 +1471,17 @@ ___diff_dump_schema() {
    IFS="$old_IFS"

    # shellcheck disable=SC2086
-    pg_dump -h "$socket_dir" -p "$port" testdb \
+    pg_dump -h "$socket_dir" -p "$port" "$database_name" \
      --schema-only --no-owner --no-privileges \
      $table_args > "$output_file" 2>/dev/null

    # shellcheck disable=SC2086
-    pg_dump -h "$socket_dir" -p "$port" testdb \
+    pg_dump -h "$socket_dir" -p "$port" "$database_name" \
      --data-only --no-owner --no-privileges \
      $table_args >> "$output_file" 2>/dev/null
  else
    # Schema only
-    pg_dump -h "$socket_dir" -p "$port" testdb \
+    pg_dump -h "$socket_dir" -p "$port" "$database_name" \
      --schema-only --no-owner --no-privileges \
      > "$output_file" 2>/dev/null
  fi
@@ -1208,7 +1490,8 @@ ___diff_dump_schema() {
 ___diff_immutable_tables() {
  local socket_dir="$1"
  local port="$2"
-  psql -h "$socket_dir" -p "$port" -d testdb -tAv ON_ERROR_STOP=1 -c "$(cat <<'SQL'
+  local database_name="${3:-${PG_DATABASE:-testdb}}"
+  psql -h "$socket_dir" -p "$port" -d "$database_name" -tAv ON_ERROR_STOP=1 -c "$(cat <<'SQL'
 SELECT n.nspname || '.' || c.relname
 FROM pg_inherits i
 JOIN pg_class    c ON c.oid = i.inhrelid
@@ -1226,8 +1509,9 @@ ___diff_immutable_data() {
  local sock2="$3"
  local port2="$4"
  local out_file="$5"
+  local database_name="${6:-${PG_DATABASE:-testdb}}"

-  if ! psql -h "$sock1" -p "$port1" -d testdb -tAc \
+  if ! psql -h "$sock1" -p "$port1" -d "$database_name" -tAc \
    "SELECT 1 FROM pg_class c JOIN pg_namespace n ON n.oid=c.relnamespace WHERE n.nspname='hectic' AND c.relname='immutable';" \
    >/dev/null 2>&1
  then
@@ -1256,10 +1540,10 @@ ___diff_immutable_data() {
    log info "  $tbl"
    : > "$data1"
    : > "$data2"
-    pg_dump -h "$sock1" -p "$port1" testdb \
+    pg_dump -h "$sock1" -p "$port1" "$database_name" \
      --data-only --no-owner --no-privileges --column-inserts -t "$tbl" \
      > "$data1" 2>/dev/null || :
-    pg_dump -h "$sock2" -p "$port2" testdb \
+    pg_dump -h "$sock2" -p "$port2" "$database_name" \
      --data-only --no-owner --no-privileges --column-inserts -t "$tbl" \
      > "$data2" 2>/dev/null || :
    {
@@ -1405,6 +1689,8 @@ subcommand_diff() {
  DIFF_TABLES="" # TODO: add cron table
  DIFF_NO_CRON=0 # TODO: useless option
  DIFF_BACKUP_PATH=""
+  DIFF_IGNORE_MIGRATION_FAIL=0
+  DIFF_DATABASE="${PG_DATABASE:-testdb}"

  while [ $# -gt 0 ]; do
    case $1 in
@@ -1420,6 +1706,10 @@ subcommand_diff() {
        DIFF_NO_CRON=1
        shift
      ;;
+      --ignore-migration-fail)
+        DIFF_IGNORE_MIGRATION_FAIL=1
+        shift
+      ;;
      -m|--mock)
        HYDRATE_USE_MOCK=1
        shift
@@ -1433,9 +1723,8 @@ subcommand_diff() {
        exit 9
      ;;
      *)
-        # NOTE: yes, RESTORE, not DIFF prefix
-        if [ -z "$RESTORE_BACKUP_PATH" ]; then
-          RESTORE_BACKUP_PATH="$1"
+        if [ -z "$DIFF_BACKUP_PATH" ]; then
+          DIFF_BACKUP_PATH="$1"
          shift
        else
          log error "diff: unexpected argument $1"
@@ -1445,14 +1734,21 @@ subcommand_diff() {
    esac
  done

+  if [ -z "$DIFF_BACKUP_PATH" ]; then
+    DIFF_BACKUP_PATH="$DEFAULT_BACKUP_PATH"
+  fi
+  if [ -f "$DIFF_BACKUP_PATH/database_name" ]; then
+    DIFF_DATABASE=$(tr -d '\n' < "$DIFF_BACKUP_PATH/database_name")
+  fi
+
  DIFF_TMPDIR="${LOCAL_DIR}/focus/database-diff-operation"
  DIFF_PGDATA1="$DIFF_TMPDIR/pgdata1"
  DIFF_PGDATA2="$DIFF_TMPDIR/pgdata2"
  # TODO: suka, logi drugie
  DIFF_PGLOGFILE1="$DIFF_PGDATA1/logfile"
  DIFF_PGLOGFILE2="$DIFF_PGDATA2/logfile"
-  DIFF_PGURL1="postgresql://localhost:5432/testdb?host=$DIFF_PGDATA1/sock"
-  DIFF_PGURL2="postgresql://localhost:5432/testdb?host=$DIFF_PGDATA2/sock"
+  DIFF_PGURL1="postgresql://localhost:5432/$DIFF_DATABASE?host=$DIFF_PGDATA1/sock"
+  DIFF_PGURL2="postgresql://localhost:5432/$DIFF_DATABASE?host=$DIFF_PGDATA2/sock"

  if [ "${DIFF_LOG+x}" ]; then
    log_pager -O "$DIFF_PGLOGFILE1" "$DIFF_PGLOGFILE2"
@@ -1474,21 +1770,26 @@ subcommand_diff() {

  log notice "provisioning ${WHITE}DB1$NC (backup + migrations)"

-  PG_WORKING_DIR="$DIFF_PGDATA1" PG_LOG_PATH="$DIFF_PGDATA1" subcommand_restore
+  PG_WORKING_DIR="$DIFF_PGDATA1" PG_LOG_PATH="$DIFF_PGDATA1" subcommand_restore "$DIFF_BACKUP_PATH"

  log info "applying migrations to ${WHITE}DB1$NC"
  subcommand_migrator migrate up all \
    --db-url \
      "$DIFF_PGURL1" \
    || {
-      log warn "migrations failed or none to apply"
+      if [ "$DIFF_IGNORE_MIGRATION_FAIL" = "1" ]; then
+        log warn "migrations failed; continuing because --ignore-migration-fail is set"
+      else
+        log error "migrations failed"
+        exit 1
+      fi
    }

  log notice "provisioning ${WHITE}DB2$NC (current sources)"

  log info "initializing ${WHITE}DB2$NC with postgres-init"
  PG_WORKING_DIR="$DIFF_PGDATA2" \
-  PG_DATABASE="testdb" \
+  PG_DATABASE="$DIFF_DATABASE" \
  PG_DISABLE_LOGGING=1 \
    postgres-init || {
    log error "failed to initialize ${WHITE}DB2$NC"
@@ -1514,8 +1815,11 @@ subcommand_diff() {
  DIFF_DUMP1="$DIFF_TMPDIR/target.sql"
  DIFF_DUMP2="$DIFF_TMPDIR/source.sql"

-  ___diff_dump_schema "$DIFF_PGDATA1/sock" "5432" "$DIFF_DUMP1" "$DIFF_TABLES"
-  ___diff_dump_schema "$DIFF_PGDATA2/sock" "5432" "$DIFF_DUMP2" "$DIFF_TABLES"
+  ___diff_dump_schema "$DIFF_PGDATA1/sock" "5432" "$DIFF_DUMP1" "$DIFF_TABLES" "$DIFF_DATABASE"
+  ___diff_dump_schema "$DIFF_PGDATA2/sock" "5432" "$DIFF_DUMP2" "$DIFF_TABLES" "$DIFF_DATABASE"
+
+  sed -i '/^\\restrict /d;/^\\unrestrict /d' "$DIFF_DUMP1" || exit 1
+  sed -i '/^\\restrict /d;/^\\unrestrict /d' "$DIFF_DUMP2" || exit 1

  # Optional: filter out cron tables
  if [ "$DIFF_NO_CRON" = "1" ]; then
@@ -1541,7 +1845,8 @@ subcommand_diff() {
  ___diff_immutable_data \
    "$DIFF_PGDATA1/sock" "5432" \
    "$DIFF_PGDATA2/sock" "5432" \
-    "$DIFF_TMPDIR/diff"
+    "$DIFF_TMPDIR/diff" \
+    "$DIFF_DATABASE"
  data_status=$?

  if [ "$schema_differs" = 0 ] && [ "$data_status" = 0 ]; then
@@ -1573,6 +1878,14 @@ if ! [ "${AS_LIBRARY+x}" ]; then
        DB_URL=$2
        shift 2
      ;;
+      normalize-backup)
+        if [ "${SUBCOMMAND+x}" ]; then
+          REMAINING_ARGS="$REMAINING_ARGS $(quote "$1")"
+        else
+          SUBCOMMAND="normalize_backup"
+        fi
+        shift
+      ;;
      deploy|replay|restore|patch|hydrate|backup|log|migrator|diff|pull_staging|test|check|cleanup|init)
        if [ "${SUBCOMMAND+x}" ]; then
          REMAINING_ARGS="$REMAINING_ARGS $(quote "$1")"
@@ -0,0 +1,222 @@
+# shellcheck shell=dash
+
+: "${SCRIPT_NAME:=$(basename "$0")}"
+
+help() {
+  # shellcheck disable=SC2059
+  printf "$(cat <<EOF
+${BGREEN}Usage:${NC} $SCRIPT_NAME [OPTIONS] <SUBCOMMAND> [OPTIONS]
+
+PostgreSQL operations utility.
+
+${BGREEN}Global Options:${NC}
+  ${BCYAN}-h${NC}, ${BCYAN}--help${NC}               Show this help message
+  ${BCYAN}-u${NC}, ${BCYAN}--url${NC} <url>         PostgreSQL connection string
+
+${BGREEN}Subcommands:${NC}
+  ${BCYAN}secrets load${NC} [OPTIONS]  Apply hectic bundle and load secrets
+
+${BGREEN}Environment:${NC}
+  ${BBLACK}PGURL${NC}                  PostgreSQL connection string fallback
+  ${BBLACK}DB_URL${NC}                 PostgreSQL connection string fallback
+  ${BBLACK}HECTIC_DOTENV_CONTENT${NC}  Raw dotenv content to load
+  ${BBLACK}HECTIC_DOTENV_FILE${NC}     Dotenv file to read when readable
+  ${BBLACK}LOCAL_DIR${NC}              Used with ENVIRONMENT for .env fallback
+  ${BBLACK}ENVIRONMENT${NC}            Falls back to ${BBLACK}\$LOCAL_DIR/.env.\$ENVIRONMENT${NC}
+
+EOF
+)"
+}
+
+help_secrets_load() {
+  # shellcheck disable=SC2059
+  printf "$(cat <<EOF
+${BGREEN}Usage:${NC} $SCRIPT_NAME ${BCYAN}secrets load${NC} [OPTIONS]
+
+Apply the hectic SQL bundle and load dotenv-backed secrets into
+${BBLACK}hectic.secret${NC}.
+
+${BGREEN}Options:${NC}
+  ${BCYAN}-h${NC}, ${BCYAN}--help${NC}               Show this help message
+  ${BCYAN}-u${NC}, ${BCYAN}--url${NC} <url>         PostgreSQL connection string
+  ${BCYAN}-f${NC}, ${BCYAN}--dotenv-file${NC} <path> Dotenv file path
+
+${BGREEN}Resolution order:${NC}
+  1. ${BBLACK}HECTIC_DOTENV_CONTENT${NC}
+  2. ${BBLACK}--dotenv-file${NC} / ${BBLACK}HECTIC_DOTENV_FILE${NC}
+  3. ${BBLACK}\$LOCAL_DIR/.env.\$ENVIRONMENT${NC}
+
+Fails with exit code ${BBLACK}3${NC} when neither a PostgreSQL URL nor a dotenv
+source can be resolved.
+
+EOF
+)"
+}
+
+resolve_pgurl() {
+  if [ -n "${PGURL:-}" ]; then
+    printf '%s' "$PGURL"
+  elif [ -n "${DB_URL:-}" ]; then
+    printf '%s' "$DB_URL"
+  else
+    return 1
+  fi
+}
+
+resolve_dotenv_content() {
+  dotenv_file="${1:-}"
+
+  if [ -n "${HECTIC_DOTENV_CONTENT:-}" ]; then
+    printf '%s' "$HECTIC_DOTENV_CONTENT"
+  elif [ -n "$dotenv_file" ]; then
+    if [ ! -f "$dotenv_file" ] || [ ! -r "$dotenv_file" ]; then
+      return 2
+    fi
+    cat "$dotenv_file"
+  elif [ -n "${ENVIRONMENT:-}" ] && [ -n "${LOCAL_DIR:-}" ] && [ -r "${LOCAL_DIR}/.env.${ENVIRONMENT}" ]; then
+    cat "${LOCAL_DIR}/.env.${ENVIRONMENT}"
+  else
+    return 1
+  fi
+}
+
+subcommand_secrets_load() {
+  change_namespace 'db ops secrets load'
+
+  dotenv_file="${HECTIC_DOTENV_FILE:-}"
+
+  while [ $# -gt 0 ]; do
+    case $1 in
+      -h|--help)
+        help_secrets_load
+        restore_namespace
+        exit 0
+      ;;
+      -u|--url)
+        if [ $# -lt 2 ]; then
+          log error "missing value for $1"
+          restore_namespace
+          exit 3
+        fi
+        PGURL=$2
+        DB_URL=$2
+        shift 2
+      ;;
+      -f|--dotenv-file)
+        if [ $# -lt 2 ]; then
+          log error "missing value for $1"
+          restore_namespace
+          exit 3
+        fi
+        dotenv_file=$2
+        shift 2
+      ;;
+      --*|-*)
+        log error "secrets load argument $1 does not exist"
+        restore_namespace
+        exit 9
+      ;;
+      *)
+        log error "secrets load subcommand $1 does not exist"
+        restore_namespace
+        exit 9
+      ;;
+    esac
+  done
+
+  if ! pgurl="$(resolve_pgurl)"; then
+    log error "PGURL or DB_URL is required"
+    restore_namespace
+    exit 3
+  fi
+
+  if dotenv_content="$(resolve_dotenv_content "$dotenv_file")"; then
+    :
+  else
+    dotenv_content_exit_code=$?
+    if [ "$dotenv_content_exit_code" -eq 2 ]; then
+      log error "dotenv file is not readable: $dotenv_file"
+    else
+      log error "dotenv source is required (HECTIC_DOTENV_CONTENT, readable dotenv file, or \$LOCAL_DIR/.env.\$ENVIRONMENT)"
+    fi
+    restore_namespace
+    exit 3
+  fi
+
+  log notice "apply hectic bundle and load secrets"
+  apply_hectic_bundle "$pgurl" "$dotenv_content"
+  restore_namespace
+}
+
+subcommand_secrets() {
+  change_namespace 'db ops secrets'
+
+  if [ $# -eq 0 ]; then
+    help_secrets_load
+    restore_namespace
+    exit 0
+  fi
+
+  subcommand=$1
+  shift
+
+  case $subcommand in
+    load)
+      subcommand_secrets_load "$@"
+    ;;
+    -h|--help)
+      help_secrets_load
+      restore_namespace
+      exit 0
+    ;;
+    *)
+      log error "secrets subcommand $subcommand does not exist"
+      restore_namespace
+      exit 9
+    ;;
+  esac
+}
+
+while [ $# -gt 0 ]; do
+  case $1 in
+    -h|--help)
+      help
+      exit 0
+    ;;
+    -u|--url)
+      if [ $# -lt 2 ]; then
+        log error "missing value for $1"
+        exit 3
+      fi
+      PGURL=$2
+      DB_URL=$2
+      shift 2
+    ;;
+    --*|-*)
+      log error "argument $1 does not exist"
+      exit 9
+    ;;
+    *)
+      break
+    ;;
+  esac
+done
+
+subcommand="${1:-}"
+
+if [ -z "$subcommand" ]; then
+  help
+  exit 0
+fi
+
+shift
+
+case $subcommand in
+  secrets)
+    subcommand_secrets "$@"
+  ;;
+  *)
+    log error "subcommand $subcommand does not exist"
+    exit 9
+  ;;
+esac
@@ -1,32 +1,14 @@
 { dash, hectic, postgresql_17, neovim, openssh, coreutils, gawk, lib, runCommand, self }:
 let
  shell = "${dash}/bin/dash";
-
-  hecticInheritanceSqlPath = ../../lib/hook/sql/hectic-inheritance.sql;
-
  hecticInheritance = runCommand "hectic-inheritance" { } ''
    mkdir -p "$out/share/hectic"
-    cp ${hecticInheritanceSqlPath} "$out/share/hectic/hectic-inheritance.sql"
+    cp ${self.lib.hectic.inheritance.path} "$out/share/hectic/hectic-inheritance.sql"
  '';

-  # Materialize the templated version SQL into the Nix store as a real file
-  # so it can be passed by path to psql -f (alongside the static siblings).
-  hecticVersionSqlFile = pkgs-writeText "hectic-version.sql" self.lib.hectic.version.sql;
-  pkgs-writeText = name: text: runCommand name { inherit text; passAsFile = [ "text" ]; } ''
-    cp "$textPath" "$out"
-  '';
+  applyBundle = self.lib.hectic.applyBundleScript;

-  hecticEnv = ''
-    HECTIC_VERSION_SQL=${hecticVersionSqlFile}
-    HECTIC_SECRET_SQL=${self.lib.hectic.secret.path}
-    HECTIC_MIGRATION_SQL=${self.lib.hectic.migration.path}
-    HECTIC_INHERITANCE_SQL=${self.lib.hectic.inheritance.path}
-    export HECTIC_VERSION_SQL HECTIC_SECRET_SQL HECTIC_MIGRATION_SQL HECTIC_INHERITANCE_SQL
-  '';
-
-  applyBundle = builtins.readFile self.lib.hectic.applyBundleScript;
-
-  mkDatabase =
+  mkDbDev =
    { postgresql ? postgresql_17 }:
    hectic.writeShellApplication {
      inherit shell;
@@ -34,19 +16,18 @@ let
        "errexit"
        "nounset"
      ];
-      excludeShellChecks = [ "SC2209" ];
+    excludeShellChecks = [ "SC1091" "SC2209" ];
      name = "database";
      runtimeInputs = [ hectic.migrator hectic.parse-uri postgresql neovim openssh coreutils gawk ];

      text = ''
-        ${builtins.readFile hectic.helpers.posix-shell.log}
-        ${builtins.readFile hectic.helpers.posix-shell.change_namespace}
-        ${builtins.readFile hectic.helpers.posix-shell.quote}
-        ${builtins.readFile hectic.helpers.posix-shell.pager_or_cat}
-        ${builtins.readFile hectic.helpers.posix-shell.with_closed_fds}
-        ${hecticEnv}
+        . ${hectic.helpers.posix-shell.log}
+        . ${hectic.helpers.posix-shell.change_namespace}
+        . ${hectic.helpers.posix-shell.quote}
+        . ${hectic.helpers.posix-shell.pager_or_cat}
+        . ${hectic.helpers.posix-shell.with_closed_fds}
        ${applyBundle}
-        ${builtins.readFile ./database.sh}
+        ${builtins.readFile ./db-dev.sh}
      '';

      meta = {
@@ -55,16 +36,42 @@ let
      };
    };

+  mkDbOps =
+    { postgresql ? postgresql_17 }:
+    hectic.writeShellApplication {
+      inherit shell;
+      bashOptions = [
+        "errexit"
+        "nounset"
+      ];
+    excludeShellChecks = [ "SC1091" "SC2209" ];
+      name = "db-ops";
+      runtimeInputs = [ postgresql coreutils ];
+
+      text = ''
+        . ${hectic.helpers.posix-shell.log}
+        . ${hectic.helpers.posix-shell.change_namespace}
+        ${applyBundle}
+        ${builtins.readFile ./db-ops.sh}
+      '';
+
+      meta = {
+        description = "PostgreSQL operations utility";
+        mainProgram = "db-ops";
+      };
+    };
+
  mkPostgresInit =
    { postgresql ? postgresql_17 }:
    hectic.writeShellApplication {
      inherit shell;
      bashOptions = [ ];
+      excludeShellChecks = [ "SC1091" ];
      name = "postgres-init";
      runtimeInputs = [ postgresql coreutils ];

      text = ''
-        ${builtins.readFile hectic.helpers.posix-shell.with_closed_fds}
+        . ${hectic.helpers.posix-shell.with_closed_fds}
        ${builtins.readFile ./postgres-init.sh}
      '';

@@ -92,7 +99,8 @@ let
    };
 in
 {
-  "db-tool"             = lib.makeOverridable mkDatabase { };
+  "db-dev"              = lib.makeOverridable mkDbDev { };
+  "db-ops"              = lib.makeOverridable mkDbOps { };
  "postgres-init"       = lib.makeOverridable mkPostgresInit { };
  "postgres-cleanup"    = lib.makeOverridable mkPostgresCleanup { };
  "hectic-inheritance"  = hecticInheritance;
--- a/Show More
+++ b/Show More
Author	SHA1	Message	Date
yukkop	09eecf5969	feat: lfs	2026-09-24 23:15:43 +00:00
yukkop	e444ea5936	fix: world-of-sosal	2026-09-23 19:26:19 +00:00
yukkop	c917e4908c	feat: zomboid backups	2026-09-22 17:15:18 +00:00
yukkop	ef849b085f	fix: wow-minecraft: finish configurations	2026-09-19 09:04:05 +00:00
yukkop	a09f247290	fix: wow-minecraft: mirror on bfs.band	2026-09-19 08:27:33 +00:00
yukkop	3a52023082	fix: configure minecraft	2026-09-19 08:13:49 +00:00
yukkop	2bd466b652	feat: configure minecraft and zomboid	2026-09-19 07:37:19 +00:00
yukkop	83cf9ff32f	.	2026-09-17 08:37:12 +00:00
yukkop	a9e538fc76	feat: prism launcher link	2026-09-16 12:48:15 +00:00
yukkop	88ec8a59d8	feat: site lessons	2026-09-15 19:48:13 +00:00
yukkop	417820544e	fix: raise runner monthly budget Ultraworked with [Sisyphus](https://github.com/code-yeongyu/oh-my-openagent) Co-authored-by: Sisyphus <clio-agent@sisyphuslabs.ai>	2026-09-15 00:58:07 +00:00
yukkop	1cba36e76e	feat: `neuro`: minecraft mods	2026-09-14 12:22:10 +00:00
yukkop	4f97fb3244	feat: `neuro`: new minecraft	2026-09-13 22:36:11 +00:00
yukkop	252d3418a5	fix: immitch 502	2026-09-13 21:49:57 +00:00
yukkop	96882fd8cd	fix: immitch 502	2026-09-13 19:11:59 +00:00
yukkop	522010b7e6	fix: minecraft	2026-09-13 18:04:18 +00:00
yukkop	761120b211	fix: anitcheat in project zomboid	2026-09-13 12:58:01 +00:00
yukkop	a04a58b792	fix: immich; feat: zomboid settings	2026-09-13 09:38:32 +00:00
yukkop	1e973e7b33	feat: configure immich	2026-09-12 22:21:33 +00:00
yukkop	66a502b1dd	fix: project zomboid module	2026-09-12 21:56:31 +00:00
yukkop	fce9ae07be	fix: configure Dify Redis connection Ultraworked with [Sisyphus](https://github.com/code-yeongyu/oh-my-openagent) Co-authored-by: Sisyphus <clio-agent@sisyphuslabs.ai>	2026-09-12 19:59:21 +00:00
yukkop	a8c4a52a68	feat: olama fix	2026-09-12 18:57:21 +00:00
yukkop	078e01c87f	feat: some	2026-09-11 20:20:57 +00:00
yukkop	6bc564de59	feat: some	2026-09-11 19:31:04 +00:00
yukkop	cc8a7cf80e	fix: reconcile healthy terminal runners	2026-09-11 09:19:04 +00:00
yukkop	279df769db	fix: retain healthy terminal runners	2026-09-11 09:19:03 +00:00
yukkop	069b18daa3	fix: check runner health before reuse	2026-09-11 09:19:03 +00:00
yukkop	37bd69e90e	fix: avoid Cargo metadata IFD	2026-09-11 09:19:03 +00:00
yukkop	572133a941	fix: avoid p4d init IFD Ultraworked with [Sisyphus](https://github.com/code-yeongyu/oh-my-openagent) Co-authored-by: Sisyphus <clio-agent@sisyphuslabs.ai>	2026-09-11 08:36:59 +00:00
yukkop	ed721dd961	fix: avoid smtp source IFD Ultraworked with [Sisyphus](https://github.com/code-yeongyu/oh-my-openagent) Co-authored-by: Sisyphus <clio-agent@sisyphuslabs.ai>	2026-09-11 08:36:46 +00:00
yukkop	4a88e7c6e8	fix: runners	2026-09-11 05:58:04 +00:00
yukkop	f0317efb6e	fix: runners not runs	2026-09-11 02:22:07 +00:00
yukkop	cb62bd3044	feat: maintain server until hour	2026-09-11 02:05:39 +00:00
yukkop	ee8a33c3b0	feat: move all workers to zero-idle	2026-09-10 22:30:58 +00:00
yukkop	0a2b96abc2	feat: highmem	2026-09-10 17:44:56 +00:00
yukkop	f286ab632c	feat: avility disable cache	2026-09-10 17:24:56 +00:00
yukkop	df0638a0f7	feat: new diagnostics	2026-09-10 15:31:13 +00:00
yukkop	b3bcd6d5a8	chore: +ssh key	2026-09-10 13:32:11 +00:00
yukkop	a4a7656459	feat: oh ah	2026-09-10 13:17:05 +00:00
yukkop	4e1c4638cd	feat: nix on runner updated	2026-09-10 12:44:06 +00:00
yukkop	baff2bc29b	feat: workflow for deploy hectic-lab	2026-09-10 12:05:46 +00:00
yukkop	76096b4221	feat: immich srorage box config	2026-09-10 11:57:21 +00:00
yukkop	bbe48685fa	feat: accurate workflow	2026-09-10 10:29:43 +00:00
yukkop	560eeae963	feat: +nix-downloaf-buffer-check	2026-09-10 10:21:50 +00:00
yukkop	3bb2b05fad	feat: +nix-downloaf-buffer-check	2026-09-10 10:19:07 +00:00
yukkop	1665aaf90a	fix: workflow neuro timeouts	2026-09-10 10:00:28 +00:00
yukkop	a100ff2ca7	feat!: migrate attic to 1mb chanks	2026-09-10 07:23:38 +00:00
yukkop	03ee994482	feat: immich	2026-09-10 06:14:29 +00:00
yukkop	9eb7eb2e5d	fix!: attic chaces with hatzner limites	2026-09-09 23:35:45 +00:00
yukkop	a5eeacd611	fix: workflow cache	2026-09-09 19:04:13 +00:00
yukkop	6dc2391b13	feat: workflow save to cache	2026-09-09 15:05:34 +00:00
yukkop	906e56a78a	fix: include nixos-rebuild in deploy	2026-09-09 12:24:00 +00:00
yukkop	aeeee27a3d	fix: pass deploy SSH config	2026-09-09 12:17:09 +00:00
yukkop	4253270da9	fix: override invalid runner cache key	2026-09-09 12:08:54 +00:00
yukkop	7580202c13	fix: use local deploy flake	2026-09-09 12:01:04 +00:00
yukkop	44fc755f41	fix: deploy neuro workflow	2026-09-09 11:36:12 +00:00
yukkop	68a4dd1900	fix: deploy neuro workflow	2026-09-09 11:23:58 +00:00
yukkop	059e2a8533	fix: deploy neuro workflow	2026-09-09 09:04:15 +00:00
yukkop	93171355a7	fix: deploy neuro workflow	2026-09-09 08:56:24 +00:00
yukkop	2045ba76ed	fix: remove microOS	2026-09-09 08:03:15 +00:00
yukkop	702f93f891	fix: microOS	2026-09-08 23:33:08 +00:00
yukkop	728814c04d	fix: pg init	2026-09-08 20:03:57 +00:00
yukkop	58d590b2be	fix: pg init	2026-09-08 19:56:06 +00:00
yukkop	beeb73736c	gross	2026-09-08 19:31:04 +00:00
yukkop	6f80f672f2	rework: fix all vulnerabilities	2026-09-08 10:36:38 +00:00
yukkop	01a0d6e00a	ci: fix: nix runners	2026-09-08 09:34:22 +00:00
yukkop	4cbd8c6921	fix: use checkout tag in nix smoke	2026-09-08 09:10:20 +00:00
yukkop	78403f6a44	ci: new workflow	2026-09-08 08:59:03 +00:00
yukkop	006ea7f52b	fix: gite nix runners	2026-09-07 18:07:59 +00:00
yukkop	8cfc7463c6	ci!: deploy workflow	2026-09-07 10:59:36 +00:00
yukkop	2fbd776fc1	feat: performace lables for gitea	2026-09-03 09:00:07 +00:00
yukkop	2ca6e3deae	feat: `hectic-lab`: volume -> /nix	2026-09-03 06:51:35 +00:00
yukkop	27ccff23f3	fix: gitea runner	2026-09-03 06:49:45 +00:00
yukkop	fd6c9536e7	fix: finaly gitea runners can works	2026-09-02 13:51:11 +00:00
yukkop	07689e1a9e	feat: p4 on hectic-lab	2026-08-31 10:51:38 +00:00
yukkop	31cac93a4c	feat: `zomboid`: server but you cannot connect to it	2026-08-27 19:37:37 +00:00
yukkop	cadc6c44be	feat: projectzomboid	2026-08-27 15:08:38 +00:00
yukkop	c5026d3e9d	feat: zero-idle gite runners infra	2026-08-27 12:34:23 +00:00
yukkop	56c033a541	feat: gitea runners infra, but it is so expancive	2026-08-25 14:44:45 +00:00
yukkop	eb16104545	fix: optimize outputs	2026-08-23 08:06:57 +00:00
yukkop	7fcf054c16	feat: `gitea`: +runner	2026-08-20 22:13:19 +00:00
yukkop	557b6e9ef0	fix: migrator	2026-07-24 15:21:19 +04:00
yukkop	e49f497045	fix: aga	2026-07-24 15:21:11 +04:00
yukkop	feb1a48db1	feat: some	2026-07-23 14:19:58 +04:00
yukkop	30732080b7	feat: some	2026-07-21 16:57:10 +04:00
yukkop	80cf1588bb	feat: +darwin config for yukkop	2026-07-19 20:03:18 +04:00
yukkop	ef7d1b29f4	feat: +njalla module	2026-07-13 23:37:14 +00:00
yukkop	7e8c6884db	feat: +plgo	2026-07-13 13:27:04 +00:00
yukkop	1dd41e608b	feat: `neuro`: stable video diffusion	2026-07-05 16:58:47 +00:00
yukkop	e41c3e5a05	ssh fixes	2026-07-04 19:32:08 +00:00
yukkop	f473280bf5	fix: matrix hardcode	2026-07-01 09:44:43 +00:00
yukkop	b08fdd6e6b	fix: matrix media	2026-07-01 09:23:26 +00:00
yukkop	f73bfc63be	fix: `zomro`: reboot	2026-06-20 22:00:52 +00:00
yukkop	f6e7c1eca9	chore	2026-06-11 14:39:57 +00:00
yukkop	6996d178ef	fix: `hetzner`: newer servers generation	2026-06-10 15:39:05 +00:00
yukkop	7f7229b199	feat: tenix host	2026-06-10 13:15:09 +00:00
yukkop	2d5bd26c36	docs: `migrator`: ~ logs & comments	2026-06-10 12:26:16 +00:00
yukkop	fba150b55b	docs: `db-tool`: ~ postgres-init and postgres-cleanup	2026-06-10 12:14:23 +00:00
yukkop	2e7bf58acf	refactor: `db-tool`: rename file	2026-06-10 11:59:21 +00:00
yukkop	b12c35f957	fix: db-tool	2026-06-10 11:40:35 +00:00
yukkop	bcf1b84dc4	fix: db-tool	2026-06-10 11:32:37 +00:00
yukkop	7c25e3b46d	feat: `db-tool`: +secrets load	2026-06-09 23:57:51 +00:00
yukkop	a20381e343	chore: ssh key	2026-06-09 22:35:06 +00:00
yukkop	bd92610a98	feat: floating ip	2026-06-09 15:43:23 +00:00
yukkop	e3ee881db6	chore: `lab`: neuro's ssh key	2026-06-09 15:11:41 +00:00
yukkop	b9eabca464	feat: `load-sops`: +generic	2026-06-09 14:51:11 +00:00
yukkop	fcc72192f5	dev: `gitea`: runners devshell	2026-06-08 10:22:34 +00:00
yukkop	5a0696ce64	ci: `gitea`: runners infra	2026-06-08 08:18:08 +00:00
yukkop	f4a59ff117	fix: `mechabellum`: 413	2026-06-07 23:14:14 +00:00
yukkop	129c82c863	chore: update mechabellum	2026-06-07 22:38:54 +00:00
yukkop	968c654320	fix: `gitea`: timeouts per iphone 429 errors	2026-06-07 22:24:11 +00:00
yukkop	98bb6c568f	chore(`package`): `gitea`: verify heatmap package build	2026-06-06 23:01:57 +00:00
yukkop	72168aa8fa	test(`package`): `gitea`: verify heatmap privacy endpoints Ultraworked with [Sisyphus](https://github.com/code-yeongyu/oh-my-openagent) Co-authored-by: Sisyphus <clio-agent@sisyphuslabs.ai>	2026-06-06 22:26:43 +00:00
yukkop	28dde5b9b1	test(`package`): `gitea`: cover heatmap author-date semantics Ultraworked with [Sisyphus](https://github.com/code-yeongyu/oh-my-openagent) Co-authored-by: Sisyphus-Junior (openai/gpt-5.5) <clio-agent@sisyphuslabs.ai>	2026-06-06 22:10:18 +00:00
yukkop	c2e0ba200c	feat(`package`): `gitea`: wire heatmap reindexing Ultraworked with [Sisyphus](https://github.com/code-yeongyu/oh-my-openagent) Co-authored-by: Sisyphus <clio-agent@sisyphuslabs.ai>	2026-06-06 22:01:59 +00:00
yukkop	2eb23ea7ea	feat(`package`): `gitea`: query heatmap by commit author date	2026-06-06 20:22:12 +00:00
yukkop	9906f71c5d	feat(`package`): `gitea`: add private heatmap opt-in	2026-06-06 20:06:21 +00:00
yukkop	593c0d9abc	feat(`package`): `gitea`: index heatmap commits by author date	2026-06-06 19:44:53 +00:00
yukkop	53dfd60b4c	feat(`package`): `gitea`: add heatmap contribution model	2026-06-06 19:20:37 +00:00
yukkop	3299daf061	feat: `db-tool`: update hook realization	2026-06-06 18:17:16 +00:00
yukkop	2856ca1d98	feat: `ente`: smpt increase security level	2026-06-06 15:34:47 +00:00
yukkop	e04b7e11da	feat: `ente`: added	2026-06-06 13:26:36 +00:00
yukkop	59dc5ecd1e	feat: `elment`: enable video messages	2026-06-06 11:09:49 +00:00
yukkop	ad6c5ab803	feat: `base`: +cache	2026-06-06 11:04:13 +00:00
yukkop	592d1f04c5	feat: `element-web`: video messages	2026-06-06 05:25:02 +00:00
yukkop	d76c0b0273	feat: vendor element-web	2026-06-05 18:55:22 +00:00
yukkop	0914391a2b	docs: atticd cache	2026-06-05 13:42:17 +00:00
yukkop	d88c1cbb4f	chore: update inputs	2026-06-05 13:12:22 +00:00
yukkop	35a5d59cbe	feat: caching	2026-06-05 13:09:18 +00:00
yukkop	341e3a0e1c	feat: atticd to S3	2026-06-05 12:42:55 +00:00
yukkop	a30d1a93dd	fix: element things	2026-06-05 10:55:51 +00:00
yukkop	c50d274ae1	feat: `lab`: +attic cache	2026-06-04 18:58:03 +00:00
yukkop	df19d16269	chore: `lab`: evgenii-kazakov email	2026-06-04 18:10:09 +00:00
yukkop	35af6720ef	fix	2026-06-03 15:03:05 +00:00
yukkop	a33432d5de	fix	2026-06-03 14:55:20 +00:00
yukkop	7152eb03de	fix	2026-06-03 13:03:29 +00:00
yukkop	8e5ba8de7f	fix	2026-06-03 12:37:44 +00:00
yukkop	7c10fda451	fix	2026-06-03 12:22:42 +00:00
yukkop	ca88f92b1a	fix	2026-06-03 09:19:12 +00:00
yukkop	5b5f119a65	fix	2026-06-03 09:08:05 +00:00
yukkop	8caf575946	fix	2026-06-03 08:54:08 +00:00
yukkop	d644d390a7	fix	2026-06-03 08:36:42 +00:00
yukkop	b56dc50e50	feat: `db-tool`: normalize-backup	2026-06-03 08:14:22 +00:00
yukkop	63223329dd	chore	2026-06-03 04:50:53 +00:00
yukkop	c74992ea85	fix: `db-tool`: generic for dbname in diff	2026-06-02 20:33:52 +00:00
yukkop	c0c024dcfd	fix: `lab`: mechabellum ssl	2026-06-02 20:33:19 +00:00
yukkop	0023e27110	feat: `lab`: disable gitea auth	2026-06-02 20:32:48 +00:00
yukkop	882b4ec871	feat: `db-tool`: fail on migration error in diff	2026-06-02 19:38:59 +00:00