fix: migrator

Ultraworked with [Sisyphus](https://github.com/code-yeongyu/oh-my-openagent)

Co-authored-by: Sisyphus <clio-agent@sisyphuslabs.ai>

.envrc

@@ -0,0 +1 @@
+use flake .#postgres-c

.gitea/workflows/runner-nix-smoke.yaml

@@ -0,0 +1,29 @@
+name: runner nix smoke
+
+on:
+  workflow_dispatch:
+  push:
+    branches:
+      - master
+    paths:
+      - .gitea/workflows/runner-nix-smoke.yaml
+      - flake.lock
+      - flake.nix
+      - infra/gitea-runners/**
+
+jobs:
+  smoke:
+    name: nix label smoke
+    runs-on: nix
+    steps:
+      - name: Nix version and cache configuration
+        run: |
+          set -eu
+          nix --version
+          nix config show substituters
+          nix config show trusted-public-keys
+
+      - name: Repository flake evaluation
+        run: |
+          set -eu
+          nix flake check --no-build

.gitea/workflows/runner-ubuntu-smoke.yaml

@@ -0,0 +1,31 @@
+name: runner ubuntu smoke
+
+on:
+  workflow_dispatch:
+  push:
+    branches:
+      - master
+    paths:
+      - .gitea/workflows/runner-ubuntu-smoke.yaml
+      - infra/gitea-runners/**
+
+jobs:
+  smoke:
+    name: ubuntu-latest label smoke
+    runs-on: ubuntu-latest
+    steps:
+      - name: Basic runner information
+        run: |
+          set -eu
+          echo "hello from gitea runner"
+          uname -a
+
+      - name: Docker smoke when available
+        run: |
+          set -eu
+          if command -v docker >/dev/null 2>&1; then
+            docker version --format 'docker client={{.Client.Version}} server={{.Server.Version}}'
+            docker run --rm hello-world
+          else
+            echo "docker command not available; skipping Docker smoke"
+          fi

.gitignore

@@ -1,4 +1,6 @@
 .env
 result
+result-*
 rust-toolchain.toml
 target/
+docs/plans

.sops.yaml

@@ -0,0 +1,87 @@
+keys:
+  - &snuff             age1w4hw2ntxrtfqhht63s9lf7nhjxjmdcc927hndn5ygcqqj532qssq4m2m6p
+  - &yukkop            age1r25zdeqq8nac6dgca9en28r57ffyz9u9d8z5yc25gc8xqz747vaqmdtk0h
+  - &yukkop-alt        age1vv46vn4hsn2lg6jy834cpu40c3mvqklldcm3hjtynrhwtpmlpc8szruz4v
+  - &nrv               age1x04u7ftjgx8de2gq596e7frauze764cmn7jjwqnx8szthvfft5qq0tezx6
+  - &bfs-server        age15yzgmsvl3ku2w863h6gw2vpmw37m9aruv6xrj4fue6n2jpm7pyuqk9xjmj
+  - &bfs-pol-server    age1fpytf05sg9n6ywpwkmn09lhpfvgtud9h75h76jhxha475zpnasqq952rpu
+  - &bfs-new-server    age17yx98qk9gzgcf2q6zhhp05p6mmtrkgz66dvyk9gqclypvlr8rersxjy5v7
+  - &neuro-server      age15yzgmsvl3ku2w863h6gw2vpmw37m9aruv6xrj4fue6n2jpm7pyuqk9xjmj
+  - &games-server      age15yzgmsvl3ku2w863h6gw2vpmw37m9aruv6xrj4fue6n2jpm7pyuqk9xjmj
+  - &hectic-lab-server age13h8twnwvgxn04l5ywtru89a6psw5d0uckr2eghxsjp88a5augvsstq5ard
+  - &umbriel-bfs       age1jxntjca8q2vxvf2jaal4xyvm2ae6sh62fhv897694kuzawfrk5asj00zdt
+
+creation_rules:
+  - path_regex: sus/home.xray.yaml$
+    key_groups:
+      - age:
+        - *yukkop
+
+  - path_regex: sus/bfs.xray.yaml$
+    key_groups:
+      - age:
+        - *snuff
+        - *yukkop
+        - *bfs-server
+        - *bfs-pol-server
+        - *bfs-new-server
+
+  - path_regex: sus/neuro.yaml$
+    key_groups:
+      - age:
+        - *yukkop
+        - *neuro-server
+
+  - path_regex: sus/games.yaml$
+    key_groups:
+      - age:
+        - *yukkop
+        - *games-server
+
+  - path_regex: sus/hectic-lab.yaml$
+    key_groups:
+      - age:
+        - *nrv
+        - *yukkop
+        - *yukkop-alt
+        - *hectic-lab-server
+        - *umbriel-bfs
+
+  - path_regex: sus/gitea-runners.yaml$
+    key_groups:
+      - age:
+        - *nrv
+        - *yukkop
+        - *yukkop-alt
+        - *hectic-lab-server
+        - *umbriel-bfs
+
+  - path_regex: sus/matrix-cluster.yaml$
+    key_groups:
+      - age:
+        - *nrv
+        - *yukkop
+        - *snuff
+        - *yukkop-alt
+        - *hectic-lab-server
+        - *bfs-pol-server
+        - *umbriel-bfs
+
+  - path_regex: sus/sentinella-default.yaml$
+    key_groups:
+      - age:
+        - *yukkop
+        - *yukkop-alt
+        - *nrv
+        - *bfs-server
+        - *bfs-pol-server
+        - *bfs-new-server
+        - *neuro-server
+        - *games-server
+        - *hectic-lab-server
+        - *umbriel-bfs
+
+  - path_regex: docs/.*\.md$
+    key_groups:
+      - age:
+        - *yukkop

darwin/system/yukkop/default.nix

@@ -0,0 +1,21 @@
+{
+  flake,
+  self,
+  inputs,
+  system ? "aarch64-darwin",
+  ...
+}: let
+  name = builtins.baseNameOf ./.;
+in inputs.nix-darwin.lib.darwinSystem {
+  inherit system;
+  specialArgs = { inherit flake self inputs; };
+  modules = [
+    inputs.home-manager.darwinModules.home-manager
+    {
+      networking.hostName = name;
+      nixpkgs.hostPlatform = system;
+      nixpkgs.overlays = [ self.overlays.default ];
+    }
+    ./${name}.nix
+  ];
+}

darwin/system/yukkop/yukkop.nix

@@ -0,0 +1,177 @@
+{
+  flake,
+  pkgs,
+  lib,
+  ...
+}: let
+  name = "yukkop";
+in {
+  system.primaryUser = name;
+  nix.settings.experimental-features = "nix-command flakes";
+
+  programs.zsh.enable = true;
+
+  services.openssh.enable = true;
+
+  users.users.${name} = {
+    home = "/Users/${name}";
+    openssh.authorizedKeys.keys = [
+      "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIJf9ljuqny71bZJokebK4Ybfml0MFMCkApS+tbMdBudp u0_a472@localhost"
+    ];
+  };
+
+  environment.systemPackages = with pkgs; [
+    aerospace
+    git
+    moreutils
+    neovim
+    tmux
+  ];
+
+  launchd.user.agents.aerospace = {
+    serviceConfig = {
+      ProgramArguments = [
+        "${pkgs.aerospace}/Applications/AeroSpace.app/Contents/MacOS/AeroSpace"
+      ];
+      RunAtLoad = true;
+      KeepAlive = true;
+      StandardOutPath = "/tmp/aerospace.out.log";
+      StandardErrorPath = "/tmp/aerospace.err.log";
+    };
+  };
+
+  home-manager.useGlobalPkgs = true;
+  home-manager.useUserPackages = true;
+  home-manager.backupFileExtension = "backup";
+  home-manager.sharedModules = [
+    (flake + "/home/module/program/tmux.nix")
+  ];
+  home-manager.users.${name} = {
+    home.stateVersion = "25.11";
+
+    home.packages = with pkgs; [
+      iproute2mac
+      jujutsu
+      ripgrep
+    ];
+
+    programs.git = {
+      enable = true;
+      lfs.enable = true;
+      settings = {
+        user.name = name;
+        user.email = "hectic.yukkop@gmail.com";
+        push.autoSetupRemote = true;
+        init.defaultBranch = "master";
+      };
+    };
+
+    programs.zsh = {
+      enable = true;
+      enableCompletion = true;
+      autosuggestion.enable = true;
+      syntaxHighlighting.enable = true;
+
+      history = {
+        size = 10000;
+        path = "$HOME/.zsh/.zsh_history";
+      };
+
+      shellAliases = {
+        drs = "darwin-rebuild switch --flake ~/pj/hearth#'yukkop|aarch64-darwin'";
+        nv = "nvim";
+        tmux = "tmux a";
+      };
+
+      initContent = ''
+        export PATH=/Users/yukkop/.opencode/bin:$PATH
+      '';
+    };
+
+    xdg.configFile."aerospace/aerospace.toml".text = ''
+      start-at-login = false
+
+      enable-normalization-flatten-containers = true
+      enable-normalization-opposite-orientation-for-nested-containers = true
+
+      default-root-container-layout = 'tiles'
+      default-root-container-orientation = 'auto'
+      accordion-padding = 30
+
+      on-focused-monitor-changed = ['move-mouse monitor-lazy-center']
+      automatically-unhide-macos-hidden-apps = false
+
+      [exec]
+        inherit-env-vars = true
+
+      [exec.env-vars]
+        PATH = '/run/current-system/sw/bin:/etc/profiles/per-user/yukkop/bin:/nix/var/nix/profiles/default/bin:/usr/local/bin:/usr/bin:/bin:/usr/sbin:/sbin:''${PATH}'
+
+      [gaps]
+        inner.horizontal = 8
+        inner.vertical = 8
+        outer.left = 8
+        outer.bottom = 8
+        outer.top = 8
+        outer.right = 8
+
+      [mode.main.binding]
+        alt-enter = 'exec-and-forget open -n /System/Applications/Utilities/Terminal.app'
+
+        alt-slash = 'layout tiles horizontal vertical'
+        alt-comma = 'layout accordion horizontal vertical'
+        alt-f = 'fullscreen'
+
+        alt-h = 'focus left'
+        alt-j = 'focus down'
+        alt-k = 'focus up'
+        alt-l = 'focus right'
+
+        alt-shift-h = 'move left'
+        alt-shift-j = 'move down'
+        alt-shift-k = 'move up'
+        alt-shift-l = 'move right'
+
+        alt-minus = 'resize smart -50'
+        alt-equal = 'resize smart +50'
+
+        alt-1 = 'workspace 1'
+        alt-2 = 'workspace 2'
+        alt-3 = 'workspace 3'
+        alt-4 = 'workspace 4'
+        alt-5 = 'workspace 5'
+        alt-6 = 'workspace 6'
+        alt-7 = 'workspace 7'
+        alt-8 = 'workspace 8'
+        alt-9 = 'workspace 9'
+
+        alt-shift-1 = 'move-node-to-workspace 1'
+        alt-shift-2 = 'move-node-to-workspace 2'
+        alt-shift-3 = 'move-node-to-workspace 3'
+        alt-shift-4 = 'move-node-to-workspace 4'
+        alt-shift-5 = 'move-node-to-workspace 5'
+        alt-shift-6 = 'move-node-to-workspace 6'
+        alt-shift-7 = 'move-node-to-workspace 7'
+        alt-shift-8 = 'move-node-to-workspace 8'
+        alt-shift-9 = 'move-node-to-workspace 9'
+
+        alt-tab = 'workspace-back-and-forth'
+        alt-shift-tab = 'move-workspace-to-monitor --wrap-around next'
+
+        alt-shift-semicolon = 'mode service'
+
+      [mode.service.binding]
+        esc = ['reload-config', 'mode main']
+        r = ['flatten-workspace-tree', 'mode main']
+        f = ['layout floating tiling', 'mode main']
+        b = ['balance-sizes', 'mode main']
+        backspace = ['close-all-windows-but-current', 'mode main']
+
+        alt-shift-h = ['join-with left', 'mode main']
+        alt-shift-j = ['join-with down', 'mode main']
+        alt-shift-k = ['join-with up', 'mode main']
+        alt-shift-l = ['join-with right', 'mode main']
+    '';
+  };
+  system.stateVersion = 6;
+}

devshell/c.nix

@@ -0,0 +1,17 @@
+{
+  system,
+  pkgs,
+  self 
+}: pkgs.mkShell {
+  buildInputs = (with pkgs; [
+      inotify-tools
+      gdb
+      gcc
+    ]) ++ (with self.packages.${system}; [
+      c-hectic
+      nvim-pager
+      watch
+    ]);
+
+    PAGER = "${self.packages.${system}.nvim-pager}/bin/pager";
+  }

devshell/default.nix

@@ -0,0 +1,31 @@
+{ system, pkgs, self, ... }: 
+(import ./dev { inherit self system pkgs; }) 
+// {
+  c          = import ./c.nix          { inherit self system pkgs; };
+  postgres-c = import ./postgres-c.nix { inherit self system pkgs; };
+  pure-c     = import ./pure-c.nix     { inherit self system pkgs; };
+  rust       = import ./rust.nix       { inherit self system pkgs; };
+  haskell    = import ./haskell.nix    { inherit self system pkgs; };
+  neuro      = import ./neuro.nix      { inherit self system pkgs; };
+  xmpp       = import ./xmpp.nix       { inherit self system pkgs; };
+  gitea-runners = import ./gitea-runners.nix { inherit pkgs; };
+  default    = pkgs.mkShell {
+    buildInputs =
+      (with self.packages.${system}; [
+        nvim-alias
+        nvim-pager
+      ])
+      ++ (with pkgs; [
+        git
+        jq
+        yq-go
+        curl
+        #(writeScriptBin "hemar-check" ''
+        #  ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null vm-postgres 'zsh -c check'
+        #'')
+      ]);
+  
+    # environment
+    PAGER = "${self.packages.${system}.nvim-pager}/bin/pager";
+  };
+}

devshell/dev/default.nix

@@ -0,0 +1,7 @@
+{
+  system,
+  self,
+  pkgs
+}: {
+  hemar      = import ./hemar      { inherit self system pkgs; };
+}

devshell/dev/hemar/default.nix

@@ -0,0 +1,8 @@
+{ pkgs, ... }: pkgs.mkShell {
+  buildInputs = (with pkgs; [
+    dash
+    (pkgs.writeShellScriptBin "letest" ''
+      ${pkgs.dash}/bin/dash ${./test.sh} "$@"
+    '')
+  ]);
+}

devshell/dev/hemar/test.sh

@@ -0,0 +1,7 @@
+#!/bin/dash
+
+ROOT_DIR="$(git rev-parse --show-toplevel 2>/dev/null || pwd)"
+
+if [ "${1:?}" = 'test' ]; then
+  dash "${ROOT_DIR}/package/hemar/test.sh"
+fi

devshell/gitea-runners.nix

@@ -0,0 +1,122 @@
+{ pkgs, ... }: let
+  opentofuUnstable = "github:NixOS/nixpkgs/nixos-unstable#opentofu";
+
+  tofu = pkgs.writeShellScriptBin "tofu" ''
+    exec ${pkgs.nix}/bin/nix run ${opentofuUnstable} -- "$@"
+  '';
+
+  giteaRunnersSetup = pkgs.writeShellScriptBin "gitea-runners-setup" /* sh */ ''
+    cat <<'EOF'
+Gitea runners setup checklist
+
+Tools available in this shell:
+  tofu, kubectl, kustomize, kubeconform, sops, age, awscli2, hcloud, tea,
+  docker, skopeo, go-containerregistry, jq, yq-go, curl, git, openssh, nix
+
+Environment expected before real deploy/apply:
+  TF_VAR_hcloud_token
+  TF_VAR_ssh_public_key
+  TF_VAR_ssh_private_key
+  S3 backend credentials and endpoint access
+  a matching SOPS age identity for sus/gitea-runners.yaml
+  kubectl access to the target cluster
+  a concrete registry digest for the pushed Nix-capable runner image if enabling
+  the nix label
+
+OpenTofu validation gate:
+  tofu version
+  tofu -chdir=infra/gitea-runners/opentofu validate
+
+Nix image build/publish/digest gate:
+  nix build .#gitea-runner-nix-image
+  publish the archive, then pin the registry-reported digest in the runner label
+  mapping
+  nix:docker://gitea.hectic-lab.com/hectic-lab/gitea-runner-nix-image@sha256:<registry-digest>
+
+SOPS token Secret creation gate:
+  kubectl apply -f infra/gitea-runners/k8s/namespace.yaml
+  umask 077
+  token_file=$(mktemp /tmp/gitea-runner-token.XXXXXX)
+  trap 'rm -f "$token_file"' EXIT
+  sops -d --extract '["gitea"]["hectic-lab"]["org-runner-registration-token"]' sus/gitea-runners.yaml > "$token_file"
+  kubectl -n gitea-runners create secret generic gitea-runner-token \
+    --from-file=token="$token_file" \
+    --dry-run=client \
+    -o yaml | kubectl -n gitea-runners apply -f -
+
+Cluster provision gate:
+  tofu -chdir=infra/gitea-runners/opentofu init
+  tofu -chdir=infra/gitea-runners/opentofu validate
+  tofu -chdir=infra/gitea-runners/opentofu plan -out=.sisyphus/evidence/task-12-deploy.plan
+  tofu -chdir=infra/gitea-runners/opentofu apply .sisyphus/evidence/task-12-deploy.plan
+  export KUBECONFIG="$(tofu -chdir=infra/gitea-runners/opentofu output -raw kubeconfig_path)"
+
+Kubernetes apply gate:
+  kubectl config current-context
+  kubectl get nodes -o wide
+  kubectl get sc
+  kubectl apply -k infra/gitea-runners/k8s
+
+Verification commands:
+  kubectl -n gitea-runners get statefulset gitea-runner
+  kubectl -n gitea-runners rollout status statefulset/gitea-runner --timeout=10m
+  kubectl -n gitea-runners get pods -l app.kubernetes.io/name=gitea-runner -o wide
+  kubectl -n gitea-runners get pvc -l app.kubernetes.io/name=gitea-runner -o wide
+  kubectl -n gitea-runners get events --sort-by=.lastTimestamp | tail -n 50
+  kubectl -n gitea-runners logs statefulset/gitea-runner -c runner --tail=200
+
+Main blockers and gates:
+  do not run tofu apply without all external inputs
+  do not apply the k8s overlay until the gitea-runner-token Secret exists
+  do not enable the nix label until the image has been published with a concrete digest
+  do not print, load, or require secrets on shell entry
+EOF
+  '';
+in pkgs.mkShell {
+  name = "gitea-runners";
+
+  buildInputs = [
+    tofu
+    giteaRunnersSetup
+    pkgs.nix
+    pkgs.kubectl
+    pkgs.kustomize
+    pkgs.kubeconform
+    pkgs.sops
+    pkgs.age
+    pkgs.awscli2
+    pkgs.hcloud
+    pkgs.tea
+    pkgs.docker
+    pkgs.skopeo
+    pkgs.go-containerregistry
+    pkgs.jq
+    pkgs.yq-go
+    pkgs.curl
+    pkgs.git
+    pkgs.openssh
+  ];
+
+  shellHook = ''
+    export GITEA_RUNNERS_ROOT="$PWD/infra/gitea-runners"
+    export GITEA_RUNNERS_TOFU_DIR="$GITEA_RUNNERS_ROOT/opentofu"
+    export GITEA_RUNNERS_K8S_DIR="$GITEA_RUNNERS_ROOT/k8s"
+    export GITEA_RUNNERS_IMAGE_DIR="$GITEA_RUNNERS_ROOT/image"
+    export GITEA_RUNNERS_NAMESPACE="gitea-runners"
+
+    alias cd-gitea-runners='cd "$GITEA_RUNNERS_ROOT"'
+    alias cd-gitea-runners-tofu='cd "$GITEA_RUNNERS_TOFU_DIR"'
+    alias cd-gitea-runners-k8s='cd "$GITEA_RUNNERS_K8S_DIR"'
+
+    echo ""
+    echo "=== Gitea runner setup DevShell ==="
+    echo ""
+    echo "Run gitea-runners-setup for the full setup checklist."
+    echo "Paths: "
+    echo "  root=$GITEA_RUNNERS_ROOT"
+    echo "  tofu=$GITEA_RUNNERS_TOFU_DIR"
+    echo "  k8s=$GITEA_RUNNERS_K8S_DIR"
+    echo "  image=$GITEA_RUNNERS_IMAGE_DIR"
+    echo ""
+  '';
+}

devshell/haskell.nix

@@ -0,0 +1,8 @@
+{
+  system,
+  pkgs,
+  self 
+}: self.devShells.${system}.default
+  // (pkgs.mkShell {
+    buildInputs = [pkgs.stack];
+  })

devshell/neuro.nix

@@ -0,0 +1,7 @@
+{
+  system,
+  pkgs,
+  self,
+}: pkgs.mkShell {
+  buildInputs = [];
+}

devshell/postgres-c.nix

@@ -0,0 +1,22 @@
+{
+  system,
+  pkgs,
+  self 
+}: pkgs.mkShell {
+  buildInputs = (with pkgs; [
+      inotify-tools
+      postgresql_15
+    ]) ++ (with self.packages.${system}; [
+      nvim-pager 
+    ]) ++ (with pkgs; [
+      gdb
+      gcc
+    ]);
+
+  PAGER = "${self.packages.${system}.nvim-pager}/bin/pager";
+
+  shellHook = ''
+    export PATH=${pkgs.gcc}/bin:$PATH
+    export PAGER="${self.packages.${system}.nvim-pager}/bin/pager"
+  '';
+}

devshell/pure-c.nix

@@ -0,0 +1,14 @@
+{
+  system,
+  pkgs,
+  self 
+}: pkgs.mkShell {
+  buildInputs = (with pkgs; [ inotify-tools ]) ++ (with self.packages.${system}; [ nvim-pager ]) ++ (with pkgs; [ gdb gcc binutils ]);
+  PAGER = "${self.packages.${system}.nvim-pager}/bin/pager";
+
+  shellHook = ''
+    export PATH=${pkgs.gcc}/bin:$PATH
+
+    export PAGER="${self.packages.${system}.nvim-pager}/bin/pager"
+  '';
+}

devshell/rust.nix

@@ -0,0 +1,17 @@
+{ 
+  self,
+  pkgs,
+  system
+}: let
+  rustToolchain =
+    if builtins.pathExists ./rust-toolchain.toml
+    then pkgs.pkgsBuildHost.rust-bin.fromRustupToolchainFile ./rust-toolchain.toml
+    else pkgs.pkgsBuildHost.rust-bin.stable."1.81.0".default;
+in
+  self.devShells.${system}.default
+  // (pkgs.mkShell {
+    nativeBuildInputs = [
+      rustToolchain
+      pkgs.pkg-config
+    ];
+  })

devshell/xmpp.nix

@@ -0,0 +1,40 @@
+{
+  system,
+  pkgs,
+  self,
+}: let
+  proxychainsConf = pkgs.writeText "proxychains.conf" ''
+    strict_chain
+    proxy_dns
+    tcp_read_time_out 15000
+    tcp_connect_time_out 8000
+    [ProxyList]
+    socks5 127.0.0.1 1080
+  '';
+
+  # Wrapper script for profanity with proxy
+  profanity-proxy = pkgs.writeShellScriptBin "profanity-proxy" ''
+    exec ${pkgs.proxychains-ng}/bin/proxychains4 -f ${proxychainsConf} ${pkgs.profanity}/bin/profanity "$@"
+  '';
+in pkgs.mkShell {
+  buildInputs = [
+    pkgs.profanity
+    pkgs.proxychains-ng
+    profanity-proxy
+  ];
+
+  shellHook = ''
+    echo ""
+    echo "=== XMPP DevShell ==="
+    echo ""
+    echo "1. Start SSH SOCKS proxy (in another terminal):"
+    echo "   ssh -D 1080 -N neuro"
+    echo ""
+    echo "2. Run profanity with proxy:"
+    echo "   profanity-proxy"
+    echo ""
+    echo "3. In profanity:"
+    echo "   /connect yukkop@accord.tube"
+    echo ""
+  '';
+}

docs/README.md

@@ -0,0 +1,3 @@
+# Documentation
+
+- [Using the `hectic` Attic Cache](./attic-cache.md)

docs/attic-cache.md

@@ -0,0 +1,237 @@
+# Using the `hectic` Attic Cache
+
+This document explains how to:
+
+1. pull build artifacts from the cache
+2. push new artifacts to the cache
+3. configure this flake to use the cache
+
+## Cache endpoints
+
+- API endpoint: `https://cache.hectic-lab.com`
+- Binary cache endpoint: `https://cache.hectic-lab.com/hectic`
+
+The `hectic` cache is:
+
+- public for reads
+- private for pushes
+
+## Requirements
+
+Use the Attic client package:
+
+```sh
+nix shell nixpkgs#attic-client
+```
+
+Or run commands directly with:
+
+```sh
+nix shell nixpkgs#attic-client -c <command>
+```
+
+## Read from the cache
+
+### Get the cache public key
+
+```sh
+nix shell nixpkgs#attic-client -c attic cache info hectic
+```
+
+Copy the `Public Key` value, which looks like:
+
+```text
+hectic:...
+```
+
+### Configure Nix to trust the cache
+
+Per-user: `~/.config/nix/nix.conf`
+
+```ini
+substituters = https://cache.nixos.org https://cache.hectic-lab.com/hectic
+trusted-public-keys = hectic:PASTE_PUBLIC_KEY_HERE
+```
+
+System-wide: `/etc/nix/nix.conf`
+
+```ini
+substituters = https://cache.nixos.org https://cache.hectic-lab.com/hectic
+trusted-public-keys = hectic:PASTE_PUBLIC_KEY_HERE
+```
+
+After that, normal Nix commands can download from the cache automatically:
+
+```sh
+nix build .#migrator
+nix develop
+nix flake check
+```
+
+## Use the cache from this flake
+
+You can also advertise the cache from `flake.nix`:
+
+```nix
+nixConfig = {
+  extra-substituters = [
+    "https://cache.nixos.org"
+    "https://cache.hectic-lab.com/hectic"
+  ];
+  extra-trusted-public-keys = [
+    "hectic:PASTE_PUBLIC_KEY_HERE"
+  ];
+};
+```
+
+Then users can run:
+
+```sh
+nix build --accept-flake-config .#migrator
+```
+
+## Log in for pushing
+
+Pushing requires an Attic token.
+
+```sh
+nix shell nixpkgs#attic-client -c attic login local https://cache.hectic-lab.com "<TOKEN>"
+```
+
+Example with `pass`:
+
+```sh
+nix shell nixpkgs#attic-client -c attic login local https://cache.hectic-lab.com "$(pass show atticd/hectic-lab/token)"
+```
+
+## Push build results
+
+### Push a package
+
+```sh
+nix build .#migrator
+nix shell nixpkgs#attic-client -c attic push local:hectic ./result
+```
+
+### Push a check
+
+```sh
+nix build .#checks.x86_64-linux.arguments
+nix shell nixpkgs#attic-client -c attic push local:hectic ./result
+```
+
+### Push a NixOS system build
+
+```sh
+nix build '.#nixosConfigurations."hectic-lab|x86_64-linux".config.system.build.toplevel'
+nix shell nixpkgs#attic-client -c attic push local:hectic ./result
+```
+
+## Recommended workflow
+
+### Local development
+
+Use the cache for reads only:
+
+```sh
+nix build .#migrator
+nix develop
+nix flake check
+```
+
+### CI / builder
+
+1. Build
+2. Push to Attic
+
+Example:
+
+```sh
+nix build .#migrator
+nix shell nixpkgs#attic-client -c attic push local:hectic ./result
+```
+
+## Useful commands
+
+### Show cache info
+
+```sh
+nix shell nixpkgs#attic-client -c attic cache info hectic
+```
+
+### Check login config
+
+```sh
+nix shell nixpkgs#attic-client -c attic cache info local:hectic
+```
+
+### Re-login with a new token
+
+```sh
+nix shell nixpkgs#attic-client -c attic login local https://cache.hectic-lab.com "<NEW_TOKEN>"
+```
+
+## Common issues
+
+### `flake 'nixpkgs' does not provide attribute 'attic'`
+
+Use:
+
+```sh
+nix shell nixpkgs#attic-client
+```
+
+Not:
+
+```sh
+nix shell nixpkgs#attic
+```
+
+### `HTTP 413 Payload Too Large`
+
+This means nginx rejected the upload body size. The server must allow large uploads on the Attic vhost.
+
+### Push succeeds for some paths but fails for others
+
+Usually means:
+
+- nginx body size limit
+- timeout/reverse proxy issue
+- bad token permissions
+
+### Cache pulls do not work
+
+Check:
+
+- `substituters`
+- `trusted-public-keys`
+- the exact public key from `attic cache info hectic`
+
+## Notes about retention and storage
+
+- The cache currently uses Hetzner Object Storage
+- If no `retention-period` is configured, cached objects do not expire automatically
+- This is good for long-lived reuse, but storage usage can grow over time
+
+## Summary
+
+### Read access
+
+```sh
+nix build .#migrator
+```
+
+after configuring:
+
+```ini
+substituters = https://cache.nixos.org https://cache.hectic-lab.com/hectic
+trusted-public-keys = hectic:PASTE_PUBLIC_KEY_HERE
+```
+
+### Push access
+
+```sh
+nix shell nixpkgs#attic-client -c attic login local https://cache.hectic-lab.com "<TOKEN>"
+nix build .#migrator
+nix shell nixpkgs#attic-client -c attic push local:hectic ./result
+```

docs/specs/sentinella-p2p-design.md

@@ -0,0 +1,101 @@
+# Spec: sentinella-p2p-design
+
+Scope: feature
+
+# sentinèlla P2P Design Spec
+
+## Goal
+Replace the hub-and-spoke sentinel topology with a fully peer-to-peer model where every node is equal.
+
+## Topology
+- Every node runs both `probe` and `watcher`
+- No privileged coordinator; any node can go down without breaking monitoring of the others
+- Duplicate Telegram alerts from multiple nodes detecting the same failure are **accepted** (reliability over deduplication)
+
+## Peer Discovery — DNS multi-A record
+- One DNS name (e.g. `peers.sentinella.com`) has multiple A records, one per node IP
+- Configured externally via any DNS registrar (Cloudflare, Namecheap, etc.)
+- Recommended TTL: **60 seconds** so new nodes propagate quickly
+- Each watcher resolves the name via `getent hosts $PEERS_DNS` on every poll cycle
+- Own IP (`$SELF`) is stripped from the result so a node never polls itself
+- No per-node DNS names needed; IP addresses are used directly in peer URLs
+
+```
+peers.sentinella.com  A  1.2.3.4    TTL 60
+peers.sentinella.com  A  5.6.7.8    TTL 60
+peers.sentinella.com  A  9.10.11.12 TTL 60
+```
+
+## Environment Variables
+
+### watcher (new, replaces sentinel)
+| Variable | Default | Required | Description |
+|---|---|---|---|
+| `PEERS_DNS` | — | yes | DNS name resolving to all peer IPs |
+| `SELF` | — | yes | This node's own IP; excluded from peer list |
+| `PEERS_PORT` | `5988` | no | Port all peers listen on |
+| `PEERS_SCHEME` | `http` | no | URL scheme for peer connections |
+| `PEERS_TOKEN` | — | no | Single Basic Auth token sent to all peers (replaces per-server TOKENS) |
+| `TG_TOKEN` | — | yes | Telegram bot token |
+| `TG_CHAT_ID` | — | yes | Telegram chat ID |
+| `TIMEOUT` | `5` | no | curl timeout seconds |
+| `POLLING_INTERVAL_SEC` | `3` | no | Seconds between poll rounds |
+| `STATE_DIR` | `/var/lib/sentinel` | no | Directory for state files |
+| `SPAM` | `0` | no | If 1, notify on every poll |
+
+### probe / router (unchanged)
+| Variable | Default | Description |
+|---|---|---|
+| `PORT` | `5988` | TCP port to listen on |
+| `URLS` | — | Space-separated URLs to health-check |
+| `VOLUMES` | all from df -P | Mount points to report |
+| `TIMEOUT` | `5` | curl timeout |
+| `AUTH_FILE` | — | Path to user:pass auth file |
+
+## Key Implementation Details
+
+### resolve_peers() in watcher.sh
+```sh
+resolve_peers() {
+  getent hosts "$PEERS_DNS" \
+    | awk '{print $1}' \
+    | grep -v "^${SELF}$" \
+    | awk -v s="$PEERS_SCHEME" -v p="$PEERS_PORT" '{print s"://"$1":"p}'
+}
+```
+Called at the top of every outer poll loop iteration — no restart needed when DNS changes.
+
+### Auth simplification
+- Old: per-server CSV `TOKENS` aligned with `SERVERS`
+- New: single optional `PEERS_TOKEN`; either all peers require auth or none do
+
+### State files
+- Unchanged: `$STATE_DIR/$(cksum url).state` contains last known state string
+- Format: `up:N/M:200` or `down:0/0:000`
+
+## Binaries
+| Old name | New name | Role |
+|---|---|---|
+| `sentinel` | `watcher` | Polls peers, sends alerts |
+| `probe` | `probe` | socat TCP listener (unchanged) |
+| `router` | `router` | HTTP handler (unchanged + auth bug fixed) |
+| `base64` | `base64` | awk base64 util (unchanged) |
+
+## NixOS Module Options
+```
+hectic.sentinella.enable            bool
+hectic.sentinella.peersDns          string   # e.g. "peers.sentinella.com"
+hectic.sentinella.self              string   # this node's own IP
+hectic.sentinella.port              int      # default 5988
+hectic.sentinella.urls              [string] # URLs for probe to health-check
+hectic.sentinella.volumes           [string] # mount points for probe
+hectic.sentinella.tgToken           string
+hectic.sentinella.tgChatId          string
+hectic.sentinella.pollingIntervalSec int     # default 3
+```
+Generates two systemd services: `sentinella-probe` and `sentinella-watcher`.
+
+## Known Bug to Fix (router.sh)
+The Basic Auth check references `$USER` and `$PASS` which are never populated.
+Fix: move `auth_ok=false` before the header loop and compare `$tok` against
+each entry in `$AUTH_TOKENS` (which is correctly populated from `AUTH_FILE`).

flake.nix

@@ -1,256 +1,134 @@
 {
   description = "yukkop's nix utilities";
+
+  nixConfig = {
+    extra-substituters = [
+      "https://cache.nixos.org"
+      "https://cache.hectic-lab.com/hectic"
+    ];
+    extra-trusted-public-keys = [
+      "hectic:KMQsKow4SoA9K2vOJlOljmx7/Zpf91Yy+5qEtxDDCzA="
+    ];
+  };
+
   inputs = {
-    nixpkgs.url = "github:NixOS/nixpkgs/nixos-24.11";
+    nixpkgs.url = "github:NixOS/nixpkgs/nixos-25.11";
     rust-overlay = {
       url = "github:oxalica/rust-overlay";
       inputs = {
         nixpkgs.follows = "nixpkgs";
       };
     };
+    deploy-rs = {
+      url = "github:serokell/deploy-rs";
+      inputs.nixpkgs.follows = "nixpkgs";
+    };
+    hyprland = {
+      url = "github:hyprwm/Hyprland";
+      inputs.nixpkgs.follows = "nixpkgs";
+    };
+    nixvim = {
+      url = "github:nix-community/nixvim/nixos-25.11";
+      inputs.nixpkgs.follows = "nixpkgs";
+    };
+    disko = {
+      url = "github:nix-community/disko";
+      inputs.nixpkgs.follows = "nixpkgs";
+    };
+    impermanence = {
+      url = "github:nix-community/impermanence";
+    };
+    home-manager = {
+      url = "github:nix-community/home-manager/release-25.11";
+      inputs.nixpkgs.follows = "nixpkgs";
+    };
+    nix-darwin = {
+      url = "github:nix-darwin/nix-darwin/nix-darwin-25.11";
+      inputs.nixpkgs.follows = "nixpkgs";
+    };
+    nixos-wsl = {
+      url = "github:nix-community/NixOS-WSL";
+      inputs.nixpkgs.follows = "nixpkgs";
+    };
+    nixos-hardware = {
+      url = "github:NixOS/nixos-hardware";
+    };
+    nixos-anywhere = {
+      url = "github:nix-community/nixos-anywhere";
+      inputs.nixpkgs.follows = "nixpkgs";
+    };
+    sops-nix = {
+      url = "github:Mic92/sops-nix";
+      inputs.nixpkgs.follows = "nixpkgs";
+    };
+    nixos-mailserver = {
+      url = "gitlab:simple-nixos-mailserver/nixos-mailserver/snm-25.11";
+      inputs.nixpkgs.follows = "nixpkgs";
+    };
+    nix-minecraft.url = "github:Infinidoge/nix-minecraft";
+    hectic-landing = {
+      # NOTE(yukkop): private repo - SSH access required.
+      # Only evaluated when nixosConfigurations."hectic-lab|x86_64-linux" is built.
+      url = "git+ssh://git@github.com/liquizz/hectic-landing.git";
+      inputs.nixpkgs.follows = "nixpkgs";
+    };
+    mechabellum-replay-analysis = {
+      # NOTE(yukkop): private repo - SSH access required.
+      # Only evaluated when nixosConfigurations."hectic-lab|x86_64-linux" is built.
+      url = "git+ssh://git@github.com/LysmiMx/mechabellum-replay-analysis.git";
+      inputs.nixpkgs.follows = "nixpkgs";
+    };
   };
 
-  outputs = { self, nixpkgs, rust-overlay }:
-  let
-    lib = nixpkgs.lib;
-    recursiveUpdate = lib.recursiveUpdate;
+  outputs = {
+    self,
+    nixpkgs,
+    rust-overlay,
+    ...
+  }@inputs: let
+    flake    = ./.;
+    self-lib = import ./lib { inherit flake self inputs; };
     
-    supportedSystems = [ "x86_64-linux" "aarch64-linux" "x86_64-darwin" ];
-
-    forSpecSystemsWithPkgs = supportedSystems: pkgOverlays: f:
-      builtins.foldl' (acc: system:
-        let
-          pkgs = import nixpkgs { 
-            inherit system;
-            overlays = pkgOverlays;
-          };
-          systemOutputs = f { system = system; pkgs = pkgs; };
-        in
-          recursiveUpdate acc systemOutputs
-      ) {} supportedSystems;
-
-    forAllSystemsWithPkgs = pkgOverlays: f: forSpecSystemsWithPkgs supportedSystems pkgOverlays f;
-
-    envErrorMessage = varName: "Error: The ${varName} environment variable is not set.";
-
-    parseEnv = import ./parse-env.nix;
-
-    dotEnv = builtins.getEnv "DOTENV";
-    minorEnvironment = 
-    if dotEnv != "" then 
-      if builtins.pathExists dotEnv then
-        parseEnv dotEnv
-      else
-        throw "${dotEnv} file not exist"
-    else 
-      if builtins.pathExists ./.env then
-        parseEnv ./.env
-      else
-        {};
-
-  in
-  forAllSystemsWithPkgs [ (import rust-overlay) ] ({ system, pkgs }:
-  {
-    packages.${system} = 
-    let
-	rust = {
-	 nativeBuildInputs = [
-	   pkgs.pkgsBuildHost.rust-bin.stable."1.81.0".default
-	   pkgs.pkg-config
-	 ];
-	 commonArgs = {
-           inherit (self.lib) cargoToml;
-           inherit (rust) nativeBuildInputs;
-         };
-      };
-    in
-    {
-      nvim-alias = pkgs.callPackage ./package/nvim-alias.nix {};
-      bolt-unpack = pkgs.callPackage ./package/bolt-unpack.nix {};
-      nvim-pager = pkgs.callPackage ./package/nvim-pager.nix {};
-      printobstacle = pkgs.callPackage ./package/printobstacle.nix {};
-      printprogress = pkgs.callPackage ./package/printprogress.nix {};
-      colorize = pkgs.callPackage ./package/colorize.nix {};
-      github.gh-tl = pkgs.callPackage ./package/github/gh-tl.nix {};
-      supabase-with-env-collection = pkgs.callPackage ./package/supabase-with-env-collection.nix {};
-      migration-name = pkgs.callPackage ./package/migration-name.nix {};
-      prettify-log = pkgs.callPackage ./package/prettify-log/default.nix rust.commonArgs;
-      pg = {
-        pg-from = pkgs.callPackage ./package/postgres/pg-from/default.nix rust.commonArgs;
-        pg-migration = pkgs.callPackage ./package/postgres/pg-migration/default.nix rust.commonArgs;
-      };
-    };
-
-    devShells.${system} = 
-    let
-      shells = self.devShells.${system};
-    in
-    {
-      default = pkgs.mkShell {
-        buildInputs = (with self.packages.${system}; [
-          nvim-alias
-	  #prettify-log
-	  nvim-pager
-        ]) ++ (with pkgs; [
-	  git
-	  jq
-	  yq-go
-	  curl
-	]);
-
-        # environment
-        PAGER="${self.packages.${system}.nvim-pager}/bin/pager";
-      };
-      rust =
+    # Create overlay that includes legacy packages
+    overlayWithLegacy = system: final: prev: 
       let
-        rustToolchain = if builtins.pathExists ./rust-toolchain.toml then
-          pkgs.pkgsBuildHost.rust-bin.fromRustupToolchainFile ./rust-toolchain.toml
-        else
-	  pkgs.pkgsBuildHost.rust-bin.stable."1.81.0".default;
-      in
-      shells.default //
-      (pkgs.mkShell {
-        nativeBuildInputs = [ 
-	   rustToolchain
-	   pkgs.pkg-config
-	];
-      });
-      haskell = shells.default // (pkgs.mkShell {
-        buildInputs = [ pkgs.stack ];
-      });
-    };
-
-
-    nixosModules.${system} = {
-      "preset.default" = { pkgs, modulesPath, ... }: {
-          imports = [
-              (modulesPath + "/profiles/qemu-guest.nix")
-	  ];
-
-          services.getty.autologinUser = "root";
-
-          programs.zsh.enable = true;
-          users.defaultUserShell = pkgs.zsh;
-
-          # Enable flakes and new 'nix' command
-	  nix.settings.experimental-features = "nix-command flakes";
-
-          virtualisation.vmVariant.virtualisation = {
-            qemu.options = [
-              "-nographic" 
-              "-display curses"
-              "-append console=ttyS0"
-              "-serial mon:stdio"
-              "-vga qxl"
-            ];
-            forwardPorts = [
-              { from = "host"; host.port = 40500; guest.port = 22; }
-            ];
-          };
-
-          services.openssh = {
-            enable = true;
-            settings = {
-              PasswordAuthentication = false;
-            };
-          };
-
-          networking.firewall = {
-            enable = true;
-            allowedTCPPorts = [ ];
-          };
-
-	  environment = {
-	    defaultPackages = [];
-            systemPackages = (with pkgs; [ 
-              curl
-              neovim
-	      yq-go
-	      jq
-	      htop-vim
-            ]) ++ (with self.packages.${system}; [
-	      prettify-log
-	      nvim-pager
-            ]);
-	    variables = {
-              PAGER=with self.packages.${system}; "${nvim-pager}/bin/pager";
-	    };
-	  };
-
-	  
-          system.stateVersion = "24.11";
-      };
-      "hardware.hetzner" = { ... }: {
-          boot.loader.grub.device = "/dev/sda";
-          boot.initrd.availableKernelModules = [
-            "ata_piix"
-            "uhci_hcd"
-            "xen_blkfront"
-            "vmw_pvscsi"
-          ];
-          boot.initrd.kernelModules = [ "nvme" ];
-          fileSystems."/" = { device = "/dev/sda1"; fsType = "ext4"; };
-      };
-    };
-  }) // {
-    overlays.default =
-    final: prev: (
-    let
-      version = "1.6.1";
-      buildHttpExt = versionSuffix: let
-          buildPostgresqlExtension =
-            prev.callPackage (import (builtins.path {
-              name = "extension-builder";
-              path = ./buildPostgresqlExtension.nix;
-            })) {
-              postgresql = prev."postgresql_${versionSuffix}";
-            };
-        in buildPostgresqlExtension {
-          pname = "http";
-	  inherit version;
-          src = prev.fetchFromGitHub {
-            owner = "pramsey";
-            repo = "pgsql-http";
-            rev  = "v${version}";
-            hash = "sha256-C8eqi0q1dnshUAZjIsZFwa5FTYc7vmATF3vv2CReWPM=";
-          };
-          nativeBuildInputs = with prev; [ pkg-config curl ];
+        baseOverlay = (import ./overlay { inherit flake self inputs nixpkgs; }) final prev;
+        legacyPackages = import ./legacy { inherit system pkgs self; };
+        pkgs = import nixpkgs {
+          inherit system;
+          overlays = [ (import rust-overlay) ];
         };
-    in
-    {
-      hectic = self.packages.${prev.system};
-      postgresql_17 = prev.postgresql_17 // { pkgs = prev.postgresql_17.pkgs // { http = buildHttpExt "17"; }; };
-      postgresql_16 = prev.postgresql_16 // { pkgs = prev.postgresql_16.pkgs // { http = buildHttpExt "16"; }; };
-      postgresql_15 = prev.postgresql_15 // { pkgs = prev.postgresql_15.pkgs // { http = buildHttpExt "15"; }; };
-      postgresql_14 = prev.postgresql_14 // { pkgs = prev.postgresql_14.pkgs // { http = buildHttpExt "14"; }; };
-    });
-    lib = {
-      # -- For all systems --
-      inherit dotEnv minorEnvironment parseEnv forAllSystemsWithPkgs forSpecSystemsWithPkgs;
-
-      makeEnvironment = envVars:
-        builtins.listToAttrs
-          (map (name: { inherit name; value = self.lib.getEnv name; }) envVars);
-
-      # -- Env processing --
-      getEnv = varName: let 
-        var = builtins.getEnv varName;
       in
-      if var != "" then
-        var
-      else if minorEnvironment ? varName then
-        minorEnvironment."${varName}"
-      else
-        throw (envErrorMessage varName);
+        baseOverlay // legacyPackages;
     
-      # -- Cargo.toml --
-      cargoToml = src: (builtins.fromTOML (builtins.readFile "${src}/Cargo.toml"));
-
-      ssh.keys = {
-          hetzner-test = {
-	      yukkop = ''ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIJ8scy1tv6zfXX6xyaukhO/fsZwif5rC89DvXNc6XxOf'';
-	  };
-      };
+    overlays = [ self.overlays.default ];
+  in self-lib.forAllSystemsWithPkgs ([(import rust-overlay)] ++ overlays) ({
+    system,
+    pkgs,
+  }: {
+    packages.${system}         = import ./package            { inherit flake self inputs pkgs system; };
+    devShells.${system}        = import ./devshell           { inherit flake self inputs pkgs system; };
+    legacyPackages.${system}   = import ./legacy             { inherit flake self inputs pkgs system; };
+    checks.${system}           = import ./test               { inherit flake self inputs pkgs system; };
+  }) // {
+    lib = self-lib;
+    overlays.default           = import ./overlay            { inherit flake self inputs; };
+    nixosModules               = import ./nixos/module       { inherit flake self inputs; };
+    templates                  = import ./template           { inherit flake self inputs; };
+    nixosConfigurations = {
+      # NOTE(yukkop): in bfs one of dependencies is shadow-4.17.4 that
+      # unsupported on aarch64-darwin
+      "bfs.netherland.xray|x86_64-linux" = import ./nixos/system/bfs.netherland.xray { inherit flake self inputs; system = "x86_64-linux"; };
+      "bfs.poland.xray|x86_64-linux"     = import ./nixos/system/bfs.poland.xray     { inherit flake self inputs; system = "x86_64-linux"; };
+      # FIXME(yukkop): some why I cannot merge nixosConfigurations from `forAllSystemsWithPkgs` with this 
+      "neuro|x86_64-linux"     = import ./nixos/system/neuro       { inherit flake self inputs; system = "x86_64-linux"; };
+      "games|x86_64-linux"     = import ./nixos/system/games       { inherit flake self inputs; system = "x86_64-linux"; };
+      "wsl|x86_64-linux"        = import ./nixos/system/wsl         { inherit flake self inputs; system = "x86_64-linux"; };
+      "tenix|x86_64-linux"      = import ./nixos/system/tenix       { inherit flake self inputs; system = "x86_64-linux"; };
+      "hectic-lab|x86_64-linux" = import ./nixos/system/hectic-lab  { inherit flake self inputs; system = "x86_64-linux"; };
+    };
+    darwinConfigurations = {
+      "yukkop|aarch64-darwin" = import ./darwin/system/yukkop { inherit flake self inputs; system = "aarch64-darwin"; };
     };
   };
 }

home/module/program/tmux.nix

@@ -0,0 +1,30 @@
+{ pkgs, ... }: {
+  programs.tmux = {
+    enable = true;
+    plugins = with pkgs.tmuxPlugins; [ resurrect continuum ];
+    keyMode = "vi";
+    escapeTime = 500;
+    historyLimit = 50000;
+    newSession = true;
+    extraConfig = ''
+      # resurrect
+      set -g @resurrect-strategy-vim 'session'
+      set -g @resurrect-strategy-nvim 'session'
+      set -g @resurrect-capture-pane-contents 'on'
+
+      resurrect_dir="$HOME/.tmux/resurrect"
+      set -g @resurrect-dir $resurrect_dir
+      set -g @resurrect-hook-post-save-all 'target=$(readlink -f $resurrect_dir/last); sed "s| --cmd .*-vim-pack-dir||g; s|/etc/profiles/per-user/$USER/bin/||g; s|/home/$USER/.nix-profile/bin/||g" $target | sponge $target'
+
+      # continuum
+      set -g @continuum-restore 'on'
+      set -g @continuum-boot 'on'
+      set -g @continuum-save-interval '10'
+
+      bind-key    -T copy-mode-vi v                  send-keys -X begin-selection
+      bind-key    -T copy-mode-vi C-v                send-keys -X rectangle-toggle
+
+      bind-key O select-pane -t :.-
+    '';
+  };
+}

infra/gitea-runners/image/README.md

@@ -0,0 +1,80 @@
+# Gitea runner Nix image
+
+The repo-owned Nix-capable job image is built by the flake package
+`gitea-runner-nix-image`.
+
+```sh
+nix build .#gitea-runner-nix-image
+```
+
+The package emits a Docker archive with the local build tag:
+
+```text
+gitea-runner-nix-image:2026-06-07
+```
+
+That tag is build metadata only. Do not use it as the final Gitea runner label
+mapping because runner job images must be immutable.
+
+## Publication target
+
+Preferred registry:
+
+```text
+gitea.hectic-lab.com/hectic-lab/gitea-runner-nix-image
+```
+
+Publish the archive without adding secrets to the image layers, then use the
+registry-reported digest as the only final `nix` label image reference:
+
+```text
+nix:docker://gitea.hectic-lab.com/hectic-lab/gitea-runner-nix-image@sha256:<registry-digest>
+```
+
+The `2026-06-07` tag may be pushed as a human-readable companion tag, but the
+runner label mapping must use the `@sha256:` reference above. Keep
+`ubuntu-latest` on the `gitea/runner` default image unless a later runner
+configuration task explicitly changes it. Only the `nix` label should select
+this custom image.
+
+If the Gitea container registry is unavailable, select a private registry that
+is reachable from the runner Kubernetes cluster and requires authentication that
+can be provided through Kubernetes image-pull secrets. Record the selected
+registry and replace the host in the same digest-pinned form:
+
+```text
+nix:docker://<private-registry>/<namespace>/gitea-runner-nix-image@sha256:<registry-digest>
+```
+
+Do not fall back to `latest` or a tag-only mapping.
+
+## Task 7 publication status
+
+Local build evidence is recorded in
+`.sisyphus/evidence/task-7-image-digest.txt`. In this environment, Docker could
+load and tag the image, but pushing to the preferred registry failed with
+`unauthorized: reqPackageAccess`, so no registry digest was available to pin as a
+concrete final mapping. Kubernetes pull smoke is recorded in
+`.sisyphus/evidence/task-7-image-pull.txt` and is blocked here because `kubectl`
+is not installed or not on `PATH`.
+
+Once registry credentials are available, rerun the push, capture the
+registry-reported digest, and replace `<registry-digest>` in the mapping above
+before Task 6/9 consumes the label configuration.
+
+## Image contents
+
+The image includes `nix`, `git`, `bash`, `coreutils`, and `cacert`. Its
+`/etc/nix/nix.conf` enables flakes and configures the repo substituters from the
+top-level `flake.nix`:
+
+```text
+experimental-features = nix-command flakes
+substituters = https://cache.nixos.org https://cache.hectic-lab.com/hectic
+trusted-public-keys = cache.nixos.org-1:6NCHdD59X431o0gW4x6l1xP+GxgH0r7u+f6p1VFlr0= hectic:KMQsKow4SoA9K2vOJlOljmx7/Zpf91Yy+5qEtxDDCzA=
+sandbox = false
+```
+
+No Gitea runner token, SSH key, SOPS key, kubeconfig, Hetzner token, or S3
+credential belongs in this image. Runtime secrets stay with the Kubernetes
+runner configuration and token-file mount contract.

infra/gitea-runners/k8s/cleanup-lifecycle.yaml

@@ -0,0 +1,154 @@
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: gitea-runner-lifecycle
+  namespace: gitea-runners
+  labels:
+    app.kubernetes.io/name: gitea-runner-lifecycle
+    app.kubernetes.io/part-of: gitea-actions
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  name: gitea-runner-lifecycle
+  namespace: gitea-runners
+  labels:
+    app.kubernetes.io/name: gitea-runner-lifecycle
+    app.kubernetes.io/part-of: gitea-actions
+rules:
+  - apiGroups:
+      - ""
+    resources:
+      - pods
+      - persistentvolumeclaims
+    verbs:
+      - get
+      - list
+  - apiGroups:
+      - apps
+    resources:
+      - statefulsets
+    verbs:
+      - get
+      - list
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: gitea-runner-lifecycle
+  namespace: gitea-runners
+  labels:
+    app.kubernetes.io/name: gitea-runner-lifecycle
+    app.kubernetes.io/part-of: gitea-actions
+subjects:
+  - kind: ServiceAccount
+    name: gitea-runner-lifecycle
+    namespace: gitea-runners
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: gitea-runner-lifecycle
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: gitea-runner-lifecycle
+  namespace: gitea-runners
+  labels:
+    app.kubernetes.io/name: gitea-runner-lifecycle
+    app.kubernetes.io/part-of: gitea-actions
+data:
+  cleanup-dry-run.sh: |
+    #!/bin/sh
+    set -eu
+
+    namespace="${RUNNER_NAMESPACE:-gitea-runners}"
+    mode="${CLEANUP_MODE:-dry-run}"
+    selector="app.kubernetes.io/name=gitea-runner"
+
+    if [ "$namespace" != "gitea-runners" ]; then
+      printf 'refusing to run outside namespace gitea-runners: %s\n' "$namespace" >&2
+      exit 13
+    fi
+
+    if [ "$mode" != "dry-run" ]; then
+      printf 'refusing destructive mode: set CLEANUP_MODE=dry-run for this CronJob\n' >&2
+      exit 13
+    fi
+
+    printf 'gitea runner lifecycle cleanup dry-run\n'
+    printf 'namespace: %s\n' "$namespace"
+    printf 'mode: %s\n\n' "$mode"
+
+    printf 'StatefulSet:\n'
+    kubectl -n "$namespace" get statefulset gitea-runner -o wide
+
+    printf '\nActive runner pods:\n'
+    kubectl -n "$namespace" get pods -l "$selector" -o wide
+
+    printf '\nRunner /data PVCs:\n'
+    kubectl -n "$namespace" get pvc -l "$selector" -o wide
+
+    printf '\nPVCs whose matching StatefulSet pod is absent (candidates only; no deletion):\n'
+    found_candidate=0
+    for pvc in $(kubectl -n "$namespace" get pvc -l "$selector" -o jsonpath='{range .items[*]}{.metadata.name}{"\n"}{end}'); do
+      pod="${pvc#data-}"
+      if ! kubectl -n "$namespace" get pod "$pod" >/dev/null 2>&1; then
+        found_candidate=1
+        printf 'candidate pvc=%s expected_pod=%s action=investigate-before-delete\n' "$pvc" "$pod"
+      fi
+    done
+
+    if [ "$found_candidate" -eq 0 ]; then
+      printf 'none\n'
+    fi
+
+    printf '\nGitea registration reconciliation:\n'
+    printf 'dry-run only: compare the pod/PVC list above with Gitea org runner registrations.\n'
+    printf 'only deregister a runner after its pod/PVC was intentionally deleted or /data/.runner was intentionally reset.\n'
+---
+apiVersion: batch/v1
+kind: CronJob
+metadata:
+  name: gitea-runner-cleanup-dry-run
+  namespace: gitea-runners
+  labels:
+    app.kubernetes.io/name: gitea-runner-lifecycle
+    app.kubernetes.io/part-of: gitea-actions
+spec:
+  schedule: "17 3 * * *"
+  concurrencyPolicy: Forbid
+  successfulJobsHistoryLimit: 3
+  failedJobsHistoryLimit: 3
+  jobTemplate:
+    spec:
+      ttlSecondsAfterFinished: 3600
+      template:
+        metadata:
+          labels:
+            app.kubernetes.io/name: gitea-runner-lifecycle
+            app.kubernetes.io/part-of: gitea-actions
+        spec:
+          serviceAccountName: gitea-runner-lifecycle
+          restartPolicy: Never
+          containers:
+            - name: cleanup-dry-run
+              image: bitnami/kubectl:1.30
+              imagePullPolicy: IfNotPresent
+              command:
+                - /bin/sh
+                - /scripts/cleanup-dry-run.sh
+              env:
+                - name: RUNNER_NAMESPACE
+                  value: gitea-runners
+                - name: CLEANUP_MODE
+                  value: dry-run
+              volumeMounts:
+                - name: lifecycle-scripts
+                  mountPath: /scripts
+                  readOnly: true
+          volumes:
+            - name: lifecycle-scripts
+              configMap:
+                name: gitea-runner-lifecycle
+                defaultMode: 0555

infra/gitea-runners/k8s/kustomization.yaml

@@ -0,0 +1,9 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+  - namespace.yaml
+  - service.yaml
+  - service-account.yaml
+  - runner-config.yaml
+  - statefulset.yaml
+  - cleanup-lifecycle.yaml

infra/gitea-runners/k8s/namespace.yaml

@@ -0,0 +1,7 @@
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: gitea-runners
+  labels:
+    app.kubernetes.io/name: gitea-runner
+    app.kubernetes.io/part-of: gitea-actions

infra/gitea-runners/k8s/runner-config.yaml

@@ -0,0 +1,36 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: gitea-runner-config
+  namespace: gitea-runners
+  labels:
+    app.kubernetes.io/name: gitea-runner
+    app.kubernetes.io/part-of: gitea-actions
+data:
+  config.yaml: |
+    log:
+      level: info
+
+    runner:
+      file: /data/.runner
+      capacity: 1
+      envs: {}
+      timeout: 3h
+      insecure: false
+      fetch_timeout: 5s
+      fetch_interval: 2s
+      labels:
+        - ubuntu-latest
+        # The nix label is intentionally disabled until the runner image has a
+        # concrete registry-reported digest; see ../runbook.md before deploy.
+
+    cache:
+      enabled: true
+      dir: /data/cache
+
+    container:
+      network: bridge
+      privileged: false
+      force_pull: true
+      valid_volumes: []
+      docker_host: unix:///runner-docker/docker.sock

infra/gitea-runners/k8s/service-account.yaml

@@ -0,0 +1,36 @@
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: gitea-runner
+  namespace: gitea-runners
+  labels:
+    app.kubernetes.io/name: gitea-runner
+    app.kubernetes.io/part-of: gitea-actions
+automountServiceAccountToken: false
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  name: gitea-runner
+  namespace: gitea-runners
+  labels:
+    app.kubernetes.io/name: gitea-runner
+    app.kubernetes.io/part-of: gitea-actions
+rules: []
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: gitea-runner
+  namespace: gitea-runners
+  labels:
+    app.kubernetes.io/name: gitea-runner
+    app.kubernetes.io/part-of: gitea-actions
+subjects:
+  - kind: ServiceAccount
+    name: gitea-runner
+    namespace: gitea-runners
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: gitea-runner

infra/gitea-runners/k8s/service.yaml

@@ -0,0 +1,16 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: gitea-runner
+  namespace: gitea-runners
+  labels:
+    app.kubernetes.io/name: gitea-runner
+    app.kubernetes.io/part-of: gitea-actions
+spec:
+  clusterIP: None
+  selector:
+    app.kubernetes.io/name: gitea-runner
+  ports:
+    - name: cache
+      port: 8088
+      targetPort: cache

infra/gitea-runners/k8s/statefulset.yaml

@@ -0,0 +1,156 @@
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+  name: gitea-runner
+  namespace: gitea-runners
+  labels:
+    app.kubernetes.io/name: gitea-runner
+    app.kubernetes.io/part-of: gitea-actions
+spec:
+  serviceName: gitea-runner
+  replicas: 5
+  podManagementPolicy: Parallel
+  selector:
+    matchLabels:
+      app.kubernetes.io/name: gitea-runner
+  template:
+    metadata:
+      labels:
+        app.kubernetes.io/name: gitea-runner
+        app.kubernetes.io/part-of: gitea-actions
+      annotations:
+        hectic-lab.com/security-note: "Privileged rootful DinD is limited to trusted internal Gitea workflows only. Do not enable untrusted fork or PR jobs for this pool."
+    spec:
+      serviceAccountName: gitea-runner
+      automountServiceAccountToken: false
+      terminationGracePeriodSeconds: 60
+      securityContext:
+        fsGroup: 1000
+      containers:
+        - name: runner
+          image: gitea/act_runner:0.2.11
+          imagePullPolicy: IfNotPresent
+          env:
+            - name: GITEA_INSTANCE_URL
+              value: https://gitea.hectic-lab.com
+            - name: GITEA_RUNNER_REGISTRATION_TOKEN_FILE
+              value: /runner-secrets/token
+            - name: CONFIG_FILE
+              value: /runner-config/config.yaml
+            - name: DOCKER_HOST
+              value: unix:///runner-docker/docker.sock
+          ports:
+            - name: cache
+              containerPort: 8088
+          resources:
+            requests:
+              cpu: 250m
+              memory: 256Mi
+            limits:
+              cpu: "1"
+              memory: 1Gi
+          livenessProbe:
+            exec:
+              command:
+                - /bin/sh
+                - -ec
+                - test -s /data/.runner && test -S /runner-docker/docker.sock
+            initialDelaySeconds: 60
+            periodSeconds: 30
+            timeoutSeconds: 5
+            failureThreshold: 6
+          readinessProbe:
+            exec:
+              command:
+                - /bin/sh
+                - -ec
+                - test -S /runner-docker/docker.sock
+            initialDelaySeconds: 15
+            periodSeconds: 10
+            timeoutSeconds: 5
+            failureThreshold: 6
+          volumeMounts:
+            - name: data
+              mountPath: /data
+            - name: config
+              mountPath: /runner-config
+              readOnly: true
+            - name: runner-token
+              mountPath: /runner-secrets
+              readOnly: true
+            - name: docker-socket
+              mountPath: /runner-docker
+        - name: docker
+          image: docker:27-dind
+          imagePullPolicy: IfNotPresent
+          args:
+            - --host=unix:///runner-docker/docker.sock
+            - --storage-driver=overlay2
+            - --tls=false
+          env:
+            - name: DOCKER_TLS_CERTDIR
+              value: ""
+            - name: DOCKER_HOST
+              value: unix:///runner-docker/docker.sock
+          securityContext:
+            # Privileged rootful DinD is intentionally scoped to this trusted
+            # internal runner pool; never expose it to untrusted fork/PR jobs.
+            privileged: true
+          resources:
+            requests:
+              cpu: 500m
+              memory: 1Gi
+            limits:
+              cpu: "2"
+              memory: 4Gi
+          livenessProbe:
+            exec:
+              command:
+                - docker
+                - info
+            initialDelaySeconds: 60
+            periodSeconds: 30
+            timeoutSeconds: 10
+            failureThreshold: 6
+          readinessProbe:
+            exec:
+              command:
+                - docker
+                - info
+            initialDelaySeconds: 20
+            periodSeconds: 10
+            timeoutSeconds: 10
+            failureThreshold: 6
+          volumeMounts:
+            - name: docker-socket
+              mountPath: /runner-docker
+            - name: docker-graph
+              mountPath: /var/lib/docker
+      volumes:
+        - name: config
+          configMap:
+            name: gitea-runner-config
+        - name: runner-token
+          secret:
+            secretName: gitea-runner-token
+            items:
+              - key: token
+                path: token
+            defaultMode: 0400
+        - name: docker-socket
+          emptyDir: {}
+        - name: docker-graph
+          emptyDir: {}
+  volumeClaimTemplates:
+    - metadata:
+        name: data
+        labels:
+          app.kubernetes.io/name: gitea-runner
+          app.kubernetes.io/part-of: gitea-actions
+      spec:
+        accessModes:
+          - ReadWriteOnce
+        storageClassName: hcloud-volumes
+        resources:
+          requests:
+            storage: 20Gi

infra/gitea-runners/opentofu/.gitignore

@@ -0,0 +1,29 @@
+# OpenTofu working directory and downloaded modules/providers.
+.terraform/
+.terraform.lock.hcl
+
+# State must live in the S3 backend for production. Local state is allowed only
+# for throwaway syntax checks with `tofu init -backend=false` and must not be
+# committed.
+terraform.tfstate
+terraform.tfstate.*
+*.tfstate
+*.tfstate.*
+crash.log
+crash.*.log
+
+# Plans can contain secrets or derived infrastructure data.
+*.tfplan
+*.plan
+kubeconfig
+kubeconfig.yaml
+*_kubeconfig.yaml
+
+# Variable files commonly carry credentials. Keep production inputs in SOPS or
+# external environment/configuration, not in checked-in files.
+*.tfvars
+*.tfvars.json
+override.tf
+override.tf.json
+*_override.tf
+*_override.tf.json

infra/gitea-runners/opentofu/README.md

@@ -0,0 +1,90 @@
+# Gitea runner OpenTofu backend contract
+
+This directory defines the safe backend, provider contract, and kube-hetzner
+cluster stack for the Gitea runner Kubernetes cluster.
+
+## Required backend
+
+Production state must use the OpenTofu S3 backend in `backend.tf`:
+
+- bucket: `gitea-runner-hectic-lab`
+- key: `gitea-runners/kube-hetzner/terraform.tfstate`
+- region: `fsn1`, aligned with the target Hetzner location
+- encryption: `encrypt = true`
+- locking: `use_lockfile = true` where the selected S3-compatible endpoint
+  supports it
+
+Before any production `tofu init`, verify the S3-compatible endpoint, credential
+source, bucket versioning, encryption behavior, and lockfile support for the
+chosen object-storage provider. Keep backend authentication externalized through
+environment variables, AWS-compatible shared config, or the production secret
+injection path from Task 3. Do not add `access_key`, `secret_key`, Hetzner
+tokens, runner tokens, kubeconfig material, or decrypted SOPS data to checked-in
+OpenTofu files.
+
+## Local state safety
+
+Production local state is forbidden. Only syntax-only validation/prototyping may
+use local state, and it must use backend-disabled initialization:
+
+```sh
+tofu -chdir=infra/gitea-runners/opentofu init -backend=false
+tofu -chdir=infra/gitea-runners/opentofu validate
+```
+
+Fail the run if production local state appears:
+
+```sh
+test ! -e infra/gitea-runners/opentofu/terraform.tfstate
+test ! -e infra/gitea-runners/opentofu/terraform.tfstate.backup
+grep -R 'backend "s3"' infra/gitea-runners/opentofu
+```
+
+The `.gitignore` in this directory blocks local state, plans, downloaded
+providers/modules, and variable files from being committed. Treat any local
+state file as disposable validation residue, never as production state.
+
+## Provider and module pins
+
+`versions.tf` pins the OpenTofu-compatible Hetzner Cloud provider to
+`hetznercloud/hcloud` version `1.60.1`. kube-hetzner research for this plan
+observed module version `2.19.3`, source `kube-hetzner/kube-hetzner/hcloud`, and
+module minimum hcloud provider requirement `>= 1.59.0`; these values are recorded
+as locals so Task 5 can wire the module without re-opening the version contract.
+
+`providers.tf` leaves the `hcloud` provider empty so authentication comes from
+the provider's external environment/config mechanisms such as `HCLOUD_TOKEN`.
+Do not set token values in `.tf` or `.tfvars` files.
+
+## Cluster shape
+
+The default cluster is deliberately fixed-size:
+
+- cluster name: `gitea-runners`
+- Hetzner location: `fsn1`
+- private network region: `eu-central`
+- control plane: one `cpx21` node in pool `control-plane`
+- workers: three `cpx31` nodes in pool `runner-workers`
+- storage: Hetzner CSI enabled with expected StorageClass `hcloud-volumes`
+- Longhorn: disabled
+- autoscaling/KEDA: not enabled in this stack
+
+The three default workers are sized for the initial five trusted privileged DinD
+jobs. To scale toward ten jobs later, keep autoscaling disabled and either raise
+`worker_count` to `5` or increase `worker_server_type`, then run a fresh
+`tofu plan` and the Task 11 Kubernetes pressure checks before applying.
+
+Required inputs must come from environment or secret injection, for example
+`TF_VAR_hcloud_token`, `TF_VAR_ssh_public_key`, and `TF_VAR_ssh_private_key`.
+Do not commit `.tfvars` files. kube-hetzner v2.19.3 writes the generated
+kubeconfig to `./<cluster_name>_kubeconfig.yaml` when `create_kubeconfig` is
+enabled; this path is ignored as operational secret material.
+
+## Known state caveat
+
+kube-hetzner may thread `hcloud_token` into Kubernetes secrets/state through its
+internal `kube_system_secrets` handling. This task does not claim that risk is
+solved. Task 5 must verify the generated plan and state before production apply
+and prove that Hetzner tokens, S3 credentials, runner tokens, kubeconfig private
+keys, and decrypted secrets are absent from committed files and unsafe state
+evidence.

infra/gitea-runners/opentofu/backend.tf

@@ -0,0 +1,16 @@
+terraform {
+  backend "s3" {
+    bucket       = "gitea-runner-hectic-lab"
+    key          = "gitea-runners/kube-hetzner/terraform.tfstate"
+    region       = "fsn1"
+    encrypt      = true
+    use_lockfile = true
+  }
+}
+
+check "remote_state_contract" {
+  assert {
+    condition     = local.production_remote_state
+    error_message = "Production OpenTofu state must use the configured S3 backend; local production state is forbidden."
+  }
+}

infra/gitea-runners/opentofu/main.tf

@@ -0,0 +1,60 @@
+locals {
+  default_storage_class = "hcloud-volumes"
+
+  control_plane_nodepools = [
+    {
+      name        = "control-plane"
+      server_type = var.control_plane_server_type
+      location    = var.hetzner_location
+      labels      = []
+      taints      = []
+      count       = 1
+    },
+  ]
+
+  agent_nodepools = [
+    {
+      name        = "runner-workers"
+      server_type = var.worker_server_type
+      location    = var.hetzner_location
+      labels      = ["node-role.hectic-lab/gitea-runner=true"]
+      taints      = []
+      count       = var.worker_count
+    },
+  ]
+}
+
+module "kube_hetzner" {
+  source  = "kube-hetzner/kube-hetzner/hcloud"
+  version = "2.19.3"
+
+  providers = {
+    hcloud = hcloud
+  }
+
+  hcloud_token    = var.hcloud_token
+  ssh_public_key  = var.ssh_public_key
+  ssh_private_key = var.ssh_private_key
+
+  cluster_name = var.cluster_name
+  base_domain  = var.base_domain
+
+  # kube-hetzner v2.19.3 writes <cluster_name>_kubeconfig.yaml; outputs below
+  # expose that expected path without outputting kubeconfig private key material.
+  create_kubeconfig = true
+
+  network_region          = var.network_region
+  load_balancer_location  = var.hetzner_location
+  control_plane_nodepools = local.control_plane_nodepools
+  agent_nodepools         = local.agent_nodepools
+
+  # Hetzner CSI is the required StorageClass provider for runner PVCs.
+  disable_hetzner_csi = false
+
+  # Longhorn is intentionally off; the initial runner PVCs use Hetzner CSI only.
+  enable_longhorn = false
+
+  # Scaling note: for 10 trusted DinD jobs later, keep autoscaling disabled and
+  # raise worker_count to 5 or increase worker_server_type after validating pod
+  # CPU, memory, and ephemeral-storage pressure in Task 11.
+}

infra/gitea-runners/opentofu/outputs.tf

@@ -0,0 +1,22 @@
+output "kubeconfig_path" {
+  description = "Path where kube-hetzner writes kubeconfig after apply. The file is operational secret material and must not be committed."
+  value       = coalesce(var.kubeconfig_path, "./${var.cluster_name}_kubeconfig.yaml")
+}
+
+output "cluster_name" {
+  description = "kube-hetzner cluster name."
+  value       = var.cluster_name
+}
+
+output "node_pool_names" {
+  description = "Control-plane and worker node pool names used by this stack."
+  value = {
+    control_plane = [for pool in local.control_plane_nodepools : pool.name]
+    workers       = [for pool in local.agent_nodepools : pool.name]
+  }
+}
+
+output "default_storage_class" {
+  description = "Default Hetzner CSI StorageClass expected for runner PVCs."
+  value       = local.default_storage_class
+}

infra/gitea-runners/opentofu/providers.tf

@@ -0,0 +1 @@
+provider "hcloud" {}

infra/gitea-runners/opentofu/variables.tf

@@ -0,0 +1,74 @@
+variable "hcloud_token" {
+  description = "Hetzner Cloud API token for kube-hetzner. Set with TF_VAR_hcloud_token or secret injection only; never commit it. kube-hetzner may place this value into Kubernetes secret resources/state, so scan plans before apply."
+  type        = string
+  sensitive   = true
+}
+
+variable "ssh_public_key" {
+  description = "SSH public key installed on cluster nodes. Supply from an external file or secret injection path."
+  type        = string
+}
+
+variable "ssh_private_key" {
+  description = "SSH private key used by kube-hetzner during bootstrap. Supply from an external file or secret injection path; never commit it."
+  type        = string
+  sensitive   = true
+}
+
+variable "cluster_name" {
+  description = "Name for the kube-hetzner runner cluster."
+  type        = string
+  default     = "gitea-runners"
+
+  validation {
+    condition     = can(regex("^[a-z0-9-]+$", var.cluster_name))
+    error_message = "cluster_name must contain only lowercase letters, numbers, and dashes."
+  }
+}
+
+variable "hetzner_location" {
+  description = "Hetzner Cloud location for all node pools. fsn1 keeps the first runner cluster in Falkenstein."
+  type        = string
+  default     = "fsn1"
+}
+
+variable "network_region" {
+  description = "Hetzner private network region. eu-central covers fsn1."
+  type        = string
+  default     = "eu-central"
+}
+
+variable "control_plane_server_type" {
+  description = "Default control-plane server type. cpx21 is small but leaves headroom for kube-system workloads."
+  type        = string
+  default     = "cpx21"
+}
+
+variable "worker_server_type" {
+  description = "Default worker server type for the initial trusted DinD runner pool. Three cpx31 workers provide enough headroom for five privileged jobs before Task 11 scaling validation."
+  type        = string
+  default     = "cpx31"
+}
+
+variable "worker_count" {
+  description = "Fixed worker count. Increase to 5 or choose a larger worker_server_type later to target 10 concurrent DinD jobs; do not enable autoscaling in this stack."
+  type        = number
+  default     = 3
+
+  validation {
+    condition     = var.worker_count >= 1
+    error_message = "worker_count must be at least 1."
+  }
+}
+
+variable "kubeconfig_path" {
+  description = "Expected kubeconfig path. kube-hetzner v2.19.3 writes this as <cluster_name>_kubeconfig.yaml when create_kubeconfig is true."
+  type        = string
+  default     = null
+}
+
+variable "base_domain" {
+  description = "Optional base domain for node reverse DNS. Empty keeps kube-hetzner defaults."
+  type        = string
+  default     = ""
+}

infra/gitea-runners/opentofu/versions.tf

@@ -0,0 +1,17 @@
+terraform {
+  required_version = ">= 1.10.1"
+
+  required_providers {
+    hcloud = {
+      source  = "hetznercloud/hcloud"
+      version = "1.60.1"
+    }
+  }
+}
+
+locals {
+  kube_hetzner_module_source  = "kube-hetzner/kube-hetzner/hcloud"
+  kube_hetzner_module_version = "2.19.3"
+  hcloud_provider_minimum     = ">= 1.59.0"
+  production_remote_state     = true
+}

infra/gitea-runners/runbook.md

@@ -0,0 +1,503 @@
+# Gitea Runner Infrastructure Runbook
+
+## Scope
+
+This directory is the repo-owned boundary for the first Gitea Actions runner
+pool. Task 1 only establishes the scaffold and immutable decision contract;
+downstream tasks will add OpenTofu backend/provider files, Kubernetes manifests,
+and a Nix-capable runner image under the existing subdirectories.
+
+The target service is `https://gitea.hectic-lab.com` for the Gitea organization
+`hectic-lab`. The first pool is fixed-size and trusted-only. "Ephemeral" means
+workflow job containers are ephemeral, while each runner pod keeps its runner
+identity in per-pod `/data/.runner` storage backed by a StatefulSet PVC.
+
+## Immutable decisions
+
+- Infrastructure is managed with OpenTofu command examples only, using the
+  `tofu` CLI.
+- Cloud provider is Hetzner; cluster bootstrap uses kube-hetzner.
+- Remote state uses the S3 backend bucket `gitea-runner-hectic-lab`.
+- Runner implementation is the non-Enterprise `gitea/runner`.
+- Runner registration uses a Gitea organization-scoped token for `hectic-lab`.
+- Runtime token delivery is SOPS-backed and mounted into the runner pod as a
+  file read through `GITEA_RUNNER_REGISTRATION_TOKEN_FILE`; plaintext token
+  environment variables are not the contract.
+- Kubernetes runner lifecycle uses a StatefulSet with one PVC per pod for
+  `/data`, including `/data/.runner`.
+- Container builds run through privileged rootful DinD inside trusted runner
+  pods; host Docker socket mounting is not an implementation path.
+- The active runner label is `ubuntu-latest`. The `nix` label is not live until
+  the Nix-capable image has been pushed and a concrete registry-reported digest
+  is added to the runner ConfigMap.
+- First scope is trusted internal workflows only, with no untrusted fork or PR
+  workflow support.
+- First scope has no autoscaling, no KEDA, and no dynamic runner controller.
+
+## Lifecycle boundaries
+
+- `infra/gitea-runners/opentofu/`: downstream OpenTofu stack for the S3 backend
+  contract, Hetzner provider configuration, and kube-hetzner module wiring.
+- `infra/gitea-runners/k8s/`: downstream namespace, ConfigMap, Secret mount,
+  StatefulSet, PVC, DinD sidecar, cleanup, and operational manifest work.
+- `infra/gitea-runners/image/`: downstream notes or sources for the runner image
+  handoff; package or flake output changes are outside Task 1.
+- `infra/gitea-runners/runbook.md`: this contract plus later operational
+  commands, rollback notes, and acceptance evidence references.
+
+## Guardrails
+
+- Enterprise ARC/actions-runner-controller are rejected alternatives and must
+  not be implemented here. Do not add ARC custom resources, controller install
+  instructions, or GitHub Actions ARC assumptions.
+- Untrusted fork/PR workflows are out of first scope; privileged DinD is only
+  acceptable for trusted internal jobs.
+- Autoscaling/KEDA is out of first scope; start with a fixed-size StatefulSet
+  runner pool.
+- No actual secrets are committed: no kubeconfig, runner token, Hetzner token,
+  S3 credentials, decrypted SOPS files, or SOPS age keys.
+- OpenTofu must not manage plaintext Kubernetes Secrets containing the Gitea
+  runner token; Kubernetes receives the token as a mounted file secret instead.
+- Do not use `localhost` or `127.0.0.1` as the Gitea URL inside job containers;
+  jobs must reach the public HTTPS service.
+
+## Initial acceptance commands
+
+Run from the repository root:
+
+```sh
+test -d infra/gitea-runners/opentofu && test -d infra/gitea-runners/k8s && test -d infra/gitea-runners/image
+test -f infra/gitea-runners/runbook.md
+grep -n "OpenTofu\|kube-hetzner\|StatefulSet\|DinD\|SOPS\|trusted" infra/gitea-runners/runbook.md
+grep -R "[E]nterprise ARC\|[a]ctions-runner-controller" infra/gitea-runners
+grep -R "[t]erraform " infra/gitea-runners || true
+grep -R "[D]ECISION NEEDED" infra/gitea-runners || true
+```
+
+Expected outcomes: the directory and file checks exit 0; the architecture-term
+grep shows this contract; ARC references appear only in the rejected-alternative
+guardrail above; there are no forbidden CLI command examples and no unresolved
+decision placeholders.
+
+## Downstream placeholders
+
+- Task 2: add OpenTofu backend/provider files and verify S3 state safety.
+- Task 3: add SOPS secret contract and runtime token delivery details.
+- Task 4: define or package the Nix-capable runner image for the `nix` label.
+- Task 5+: provision kube-hetzner, add Kubernetes resources, verify workflows,
+  and document cleanup, rollback, and scaling operations.
+
+## Runner lifecycle cleanup
+
+All lifecycle commands are scoped to the runner namespace:
+
+```sh
+kubectl -n gitea-runners get statefulset gitea-runner
+kubectl -n gitea-runners get pods -l app.kubernetes.io/name=gitea-runner -o wide
+kubectl -n gitea-runners get pvc -l app.kubernetes.io/name=gitea-runner -o wide
+```
+
+The scheduled cleanup manifest is dry-run only. It lists the StatefulSet, active
+runner pods, runner PVCs, and PVCs whose expected StatefulSet pod is absent. It
+does not delete pods, PVCs, Docker data, or Gitea runner registrations.
+
+Run the same inventory on demand without waiting for the schedule:
+
+```sh
+kubectl -n gitea-runners create job gitea-runner-cleanup-dry-run-manual --from=cronjob/gitea-runner-cleanup-dry-run
+kubectl -n gitea-runners wait --for=condition=complete job/gitea-runner-cleanup-dry-run-manual --timeout=2m
+kubectl -n gitea-runners logs job/gitea-runner-cleanup-dry-run-manual -c cleanup-dry-run
+```
+
+The cleanup job template has `ttlSecondsAfterFinished: 3600`, so completed
+manual dry-run jobs are garbage-collected by Kubernetes instead of requiring an
+operator to remove finished jobs manually.
+
+If a PVC such as `data-gitea-runner-3` is intentionally deleted, the matching
+pod loses `/data/.runner`. That runner identity must then be deregistered from
+Gitea or the replacement pod must be allowed to re-register intentionally with
+the current organization runner token. Do not delete an active runner PVC as a
+normal cleanup step.
+
+Non-UI Gitea registration reconciliation uses the Gitea API with a separate
+admin token. Store that token outside this repository and pass it as a file; do
+not print it:
+
+```sh
+kubectl -n gitea-runners create secret generic gitea-runner-admin-token --from-file=token=/secure/path/gitea-admin-token
+kubectl -n gitea-runners run gitea-runner-registration-dry-run \
+  --restart=Never \
+  --image=curlimages/curl:8.10.1 \
+  --overrides='{"spec":{"containers":[{"name":"gitea-runner-registration-dry-run","image":"curlimages/curl:8.10.1","command":["/bin/sh","-ec","umask 077; cfg=$(mktemp); trap '\''rm -f \"$cfg\"'\'' EXIT; { printf '\''header = \"Authorization: token '\''; cat /admin-token/token; printf '\''\"\\n'\''; printf '\''url = \"https://gitea.hectic-lab.com/api/v1/orgs/hectic-lab/actions/runners\"\\n'\''; } > \"$cfg\"; curl -fsS --config \"$cfg\""],"volumeMounts":[{"name":"admin-token","mountPath":"/admin-token","readOnly":true}]}],"volumes":[{"name":"admin-token","secret":{"secretName":"gitea-runner-admin-token","defaultMode":256}}]}}'
+kubectl -n gitea-runners logs pod/gitea-runner-registration-dry-run
+```
+
+Delete the temporary `gitea-runner-admin-token` Secret only after the dry-run pod
+has completed and its logs have been collected. Do not keep this admin token in
+the runner namespace longer than the reconciliation window.
+
+Only remove a stale Gitea runner registration after the corresponding pod/PVC
+was intentionally deleted or `/data/.runner` was intentionally reset. Prefer a
+Gitea CLI/API deletion from the Gitea server or an admin workstation; manual UI
+cleanup is a fallback, not the only path. Record the removed runner name and the
+Kubernetes PVC/pod deletion that made it stale.
+
+After the dry-run list identifies a stale registration and the PVC/pod deletion
+has been recorded, remove that exact Gitea runner by id through the API:
+
+```sh
+runner_id='REPLACE_WITH_STALE_RUNNER_ID'
+umask 077
+curl_config=$(mktemp /tmp/gitea-runner-admin-curl.XXXXXX)
+trap 'rm -f "$curl_config"' EXIT
+{
+  printf 'request = "DELETE"\n'
+  printf 'header = "Authorization: token '
+  cat /secure/path/gitea-admin-token
+  printf '"\n'
+  printf 'url = "https://gitea.hectic-lab.com/api/v1/orgs/hectic-lab/actions/runners/%s"\n' "$runner_id"
+} > "$curl_config"
+curl -fsS --config "$curl_config"
+```
+
+Do not run the delete command for a runner that still has an active
+`gitea-runner-*` pod or a retained `data-gitea-runner-*` PVC unless that PVC is
+being intentionally reset for re-registration.
+
+## Docker-in-Docker storage cleanup
+
+Docker layers live inside each DinD sidecar at `/var/lib/docker`, backed by the
+pod-local `docker-graph` `emptyDir`; the host Docker socket is not used. Always
+list disk usage before pruning, and run the command only against the `docker`
+container in runner pods in `gitea-runners`. Because this storage is pod-local,
+loop over pods for pool-wide cleanup:
+
+```sh
+for pod in $(kubectl -n gitea-runners get pods -l app.kubernetes.io/name=gitea-runner -o jsonpath='{range .items[*]}{.metadata.name}{"\n"}{end}'); do
+  kubectl -n gitea-runners exec "pod/${pod}" -c docker -- docker system df
+done
+
+for pod in $(kubectl -n gitea-runners get pods -l app.kubernetes.io/name=gitea-runner -o jsonpath='{range .items[*]}{.metadata.name}{"\n"}{end}'); do
+  kubectl -n gitea-runners exec "pod/${pod}" -c docker -- docker system prune --all --force --filter until=24h
+done
+
+for pod in $(kubectl -n gitea-runners get pods -l app.kubernetes.io/name=gitea-runner -o jsonpath='{range .items[*]}{.metadata.name}{"\n"}{end}'); do
+  kubectl -n gitea-runners exec "pod/${pod}" -c docker -- docker system df
+done
+```
+
+For one pod, replace the StatefulSet target with the pod name:
+
+```sh
+kubectl -n gitea-runners exec pod/gitea-runner-0 -c docker -- docker system df
+kubectl -n gitea-runners exec pod/gitea-runner-0 -c docker -- docker system prune --all --force --filter until=24h
+```
+
+Do not run host-level Docker cleanup commands and do not mount or prune a host
+Docker socket. If a pod is deleted, its `emptyDir` Docker graph is removed by
+Kubernetes; the `/data` PVC remains and still controls runner identity.
+
+## Token rotation
+
+Rotate the Gitea organization runner token without printing decrypted values:
+
+```sh
+sops sus/gitea-runners.yaml
+umask 077
+token_file=$(mktemp /tmp/gitea-runner-token.XXXXXX)
+trap 'rm -f "$token_file"' EXIT
+sops -d --extract '["gitea"]["hectic-lab"]["org-runner-registration-token"]' sus/gitea-runners.yaml > "$token_file"
+kubectl -n gitea-runners create secret generic gitea-runner-token \
+  --from-file=token="$token_file" \
+  --dry-run=client \
+  -o yaml | kubectl -n gitea-runners apply -f -
+kubectl -n gitea-runners rollout restart statefulset/gitea-runner
+kubectl -n gitea-runners rollout status statefulset/gitea-runner --timeout=10m
+kubectl -n gitea-runners get pods -l app.kubernetes.io/name=gitea-runner -o wide
+kubectl -n gitea-runners logs statefulset/gitea-runner -c runner --tail=200 | grep -Eq 'token|GITEA_RUNNER_REGISTRATION_TOKEN' && exit 1 || true
+```
+
+The `rollout restart` command above is the controlled restart path for this
+StatefulSet. Observe the rollout and each ordinal until all replacement pods are
+Ready; do not delete runner pods directly as part of normal token rotation:
+
+```sh
+kubectl -n gitea-runners wait --for=condition=Ready pod/gitea-runner-0 --timeout=5m
+kubectl -n gitea-runners wait --for=condition=Ready pod/gitea-runner-1 --timeout=5m
+kubectl -n gitea-runners wait --for=condition=Ready pod/gitea-runner-2 --timeout=5m
+kubectl -n gitea-runners wait --for=condition=Ready pod/gitea-runner-3 --timeout=5m
+kubectl -n gitea-runners wait --for=condition=Ready pod/gitea-runner-4 --timeout=5m
+kubectl -n gitea-runners get pods -l app.kubernetes.io/name=gitea-runner -o wide
+```
+
+Verification must confirm the token file mount remains present while the token
+value never appears in logs or evidence:
+
+```sh
+kubectl -n gitea-runners describe pod gitea-runner-0 | grep -n '/runner-secrets\|gitea-runner-token'
+kubectl -n gitea-runners logs pod/gitea-runner-0 -c runner --tail=200 | grep -Eq 'token|GITEA_RUNNER_REGISTRATION_TOKEN' && exit 1 || true
+```
+
+## Deploy and status
+
+These commands are executable only when the external inputs are available:
+
+- `TF_VAR_hcloud_token`
+- `TF_VAR_ssh_public_key`
+- `TF_VAR_ssh_private_key`
+- S3 backend credentials and endpoint access
+- a matching SOPS age identity for `sus/gitea-runners.yaml`
+- `kubectl` access to the target cluster
+- a concrete digest for the pushed Nix-capable runner image, if enabling the
+  `nix` label
+
+If any input is missing, stop before `tofu apply`. Do not guess values or reuse
+stale kubeconfig files.
+
+Before production Kubernetes apply or rollout, satisfy both manifest gates:
+
+1. Create or update the `gitea-runner-token` Secret from SOPS. The active
+   Kustomize overlay intentionally does not include a placeholder Secret, but
+   the StatefulSet still mounts `secretName: gitea-runner-token` as
+   `/runner-secrets/token` for `GITEA_RUNNER_REGISTRATION_TOKEN_FILE`.
+2. Keep the active ConfigMap on `ubuntu-latest` only unless the Nix-capable
+   image has been pushed successfully. Enable the `nix` label only by adding a
+   digest-pinned `docker://` mapping with the exact registry-reported sha256
+   digest from that push.
+
+Use the same SOPS materialization pattern as token rotation before applying the
+Kubernetes overlay. Applying the namespace alone is allowed so the Secret has a
+target namespace; the full overlay remains gated on the Secret and digest
+decisions:
+
+```sh
+kubectl apply -f infra/gitea-runners/k8s/namespace.yaml
+umask 077
+token_file=$(mktemp /tmp/gitea-runner-token.XXXXXX)
+trap 'rm -f "$token_file"' EXIT
+sops -d --extract '["gitea"]["hectic-lab"]["org-runner-registration-token"]' sus/gitea-runners.yaml > "$token_file"
+kubectl -n gitea-runners create secret generic gitea-runner-token \
+  --from-file=token="$token_file" \
+  --dry-run=client \
+  -o yaml | kubectl -n gitea-runners apply -f -
+```
+
+Do not run `kubectl apply -k infra/gitea-runners/k8s` until the Secret command
+above succeeds. Do not claim or enable the `nix` runner label until the image
+publication step has produced the concrete digest.
+
+```sh
+tofu -chdir=infra/gitea-runners/opentofu init
+tofu -chdir=infra/gitea-runners/opentofu validate
+tofu -chdir=infra/gitea-runners/opentofu plan -out=.sisyphus/evidence/task-12-deploy.plan
+tofu -chdir=infra/gitea-runners/opentofu apply .sisyphus/evidence/task-12-deploy.plan
+export KUBECONFIG="$(tofu -chdir=infra/gitea-runners/opentofu output -raw kubeconfig_path)"
+kubectl config current-context
+kubectl get nodes -o wide
+kubectl get sc
+kubectl apply -k infra/gitea-runners/k8s
+kubectl -n gitea-runners get statefulset gitea-runner
+kubectl -n gitea-runners rollout status statefulset/gitea-runner --timeout=10m
+kubectl -n gitea-runners get pods -l app.kubernetes.io/name=gitea-runner -o wide
+kubectl -n gitea-runners get pvc -l app.kubernetes.io/name=gitea-runner -o wide
+kubectl -n gitea-runners get events --sort-by=.lastTimestamp | tail -n 50
+kubectl -n gitea-runners logs statefulset/gitea-runner -c runner --tail=200
+```
+
+Expected status after deploy:
+
+- `kubectl config current-context` names the runner cluster context.
+- `kubectl get nodes -o wide` shows all expected Hetzner nodes Ready.
+- `kubectl get sc` shows the Hetzner CSI storage class used by runner PVCs.
+- `kubectl -n gitea-runners get statefulset gitea-runner` shows 5 desired and 5 ready replicas.
+- `kubectl -n gitea-runners get pvc` shows 5 Bound PVCs.
+- `kubectl -n gitea-runners logs statefulset/gitea-runner -c runner --tail=200` shows the runner daemon started and no token value.
+
+## Scale 5 to 10 to 5
+
+Scaling is a temporary capacity exercise, not the steady-state setting. Scale up,
+wait for readiness, run the concurrent smoke jobs, then scale back down to 5.
+
+```sh
+kubectl -n gitea-runners scale statefulset/gitea-runner --replicas=10
+kubectl -n gitea-runners rollout status statefulset/gitea-runner --timeout=10m
+kubectl -n gitea-runners get pods -l app.kubernetes.io/name=gitea-runner -o wide
+kubectl -n gitea-runners get pvc -l app.kubernetes.io/name=gitea-runner -o wide
+
+# Run the concurrent smoke workflows now.
+
+kubectl -n gitea-runners scale statefulset/gitea-runner --replicas=5
+kubectl -n gitea-runners rollout status statefulset/gitea-runner --timeout=10m
+kubectl -n gitea-runners get pods -l app.kubernetes.io/name=gitea-runner -o wide
+kubectl -n gitea-runners get pvc -l app.kubernetes.io/name=gitea-runner -o wide
+```
+
+After scaling back down, inspect the cleanup dry-run and deregister any stale
+runner registrations only for pods or PVCs that were intentionally removed.
+
+## Cleanup and stale runner deregistration
+
+Use the dry-run cleanup job to list the StatefulSet, active pods, PVCs, and any
+PVC candidates whose pod is gone. It must not delete active resources.
+
+```sh
+kubectl -n gitea-runners create job gitea-runner-cleanup-dry-run-manual --from=cronjob/gitea-runner-cleanup-dry-run
+kubectl -n gitea-runners wait --for=condition=complete job/gitea-runner-cleanup-dry-run-manual --timeout=2m
+kubectl -n gitea-runners logs job/gitea-runner-cleanup-dry-run-manual -c cleanup-dry-run
+```
+
+Pool-wide DinD storage checks and cleanup:
+
+```sh
+for pod in $(kubectl -n gitea-runners get pods -l app.kubernetes.io/name=gitea-runner -o jsonpath='{range .items[*]}{.metadata.name}{"\n"}{end}'); do
+  kubectl -n gitea-runners exec "pod/${pod}" -c docker -- docker system df
+done
+
+for pod in $(kubectl -n gitea-runners get pods -l app.kubernetes.io/name=gitea-runner -o jsonpath='{range .items[*]}{.metadata.name}{"\n"}{end}'); do
+  kubectl -n gitea-runners exec "pod/${pod}" -c docker -- docker system prune --all --force --filter until=24h
+done
+
+for pod in $(kubectl -n gitea-runners get pods -l app.kubernetes.io/name=gitea-runner -o jsonpath='{range .items[*]}{.metadata.name}{"\n"}{end}'); do
+  kubectl -n gitea-runners exec "pod/${pod}" -c docker -- docker system df
+done
+```
+
+If a PVC such as `data-gitea-runner-3` is intentionally deleted, the matching
+pod loses `/data/.runner`. Deregister that runner from Gitea, or let the
+replacement pod re-register intentionally with the current organization token.
+Never delete an active runner PVC as routine cleanup.
+
+Non-UI Gitea registration reconciliation uses an admin token stored outside this
+repository:
+
+```sh
+kubectl -n gitea-runners create secret generic gitea-runner-admin-token --from-file=token=/secure/path/gitea-admin-token
+kubectl -n gitea-runners run gitea-runner-registration-dry-run \
+  --restart=Never \
+  --image=curlimages/curl:8.10.1 \
+  --overrides='{"spec":{"containers":[{"name":"gitea-runner-registration-dry-run","image":"curlimages/curl:8.10.1","command":["/bin/sh","-ec","umask 077; cfg=$(mktemp); trap '\''rm -f \"$cfg\"'\'' EXIT; { printf '\''header = \"Authorization: token '\''; cat /admin-token/token; printf '\''\"\\n'\''; printf '\''url = \"https://gitea.hectic-lab.com/api/v1/orgs/hectic-lab/actions/runners\"\\n'\''; } > \"$cfg\"; curl -fsS --config \"$cfg\""],"volumeMounts":[{"name":"admin-token","mountPath":"/admin-token","readOnly":true}]}],"volumes":[{"name":"admin-token","secret":{"secretName":"gitea-runner-admin-token","defaultMode":256}}]}}'
+kubectl -n gitea-runners logs pod/gitea-runner-registration-dry-run
+```
+
+Delete the temporary `gitea-runner-admin-token` Secret only after the dry-run
+pod has completed and its logs have been collected.
+
+After the dry-run list identifies a stale registration and the PVC or pod
+deletion has been recorded, remove that exact Gitea runner by id through the
+API:
+
+```sh
+runner_id='REPLACE_WITH_STALE_RUNNER_ID'
+umask 077
+curl_config=$(mktemp /tmp/gitea-runner-admin-curl.XXXXXX)
+trap 'rm -f "$curl_config"' EXIT
+{
+  printf 'request = "DELETE"\n'
+  printf 'header = "Authorization: token '
+  cat /secure/path/gitea-admin-token
+  printf '"\n'
+  printf 'url = "https://gitea.hectic-lab.com/api/v1/orgs/hectic-lab/actions/runners/%s"\n' "$runner_id"
+} > "$curl_config"
+curl -fsS --config "$curl_config"
+```
+
+Do not run the delete command for a runner that still has an active
+`gitea-runner-*` pod or retained `data-gitea-runner-*` PVC unless that PVC is
+being intentionally reset for re-registration.
+
+## Application rollback
+
+Rollback the app layer only. Do not use this section to destroy the cluster.
+
+```sh
+kubectl -n gitea-runners rollout history statefulset/gitea-runner
+kubectl -n gitea-runners rollout undo statefulset/gitea-runner --to-revision=<known-good-revision>
+kubectl -n gitea-runners rollout status statefulset/gitea-runner --timeout=10m
+kubectl -n gitea-runners get pods -l app.kubernetes.io/name=gitea-runner -o wide
+kubectl -n gitea-runners get pvc -l app.kubernetes.io/name=gitea-runner -o wide
+kubectl -n gitea-runners logs statefulset/gitea-runner -c runner --tail=200
+```
+
+If a manifest rollback is needed, reapply the repo overlay after checking out the
+known-good revision, then re-run the rollout checks:
+
+```sh
+kubectl -n gitea-runners apply -k infra/gitea-runners/k8s
+kubectl -n gitea-runners rollout status statefulset/gitea-runner --timeout=10m
+```
+
+## Full cluster teardown
+
+This destroys Hetzner resources owned by the kube-hetzner stack, including the
+`gitea-runners` cluster nodes, the `control-plane` node pool, the
+`runner-workers` node pool, the cluster network, load balancer resources,
+firewall objects, and any attached Hetzner CSI volumes still managed by the
+stack. Do not run teardown unless the destruction is intentional.
+
+```sh
+tofu -chdir=infra/gitea-runners/opentofu plan -destroy -out=.sisyphus/evidence/task-12-destroy.plan
+tofu -chdir=infra/gitea-runners/opentofu show -no-color .sisyphus/evidence/task-12-destroy.plan
+tofu -chdir=infra/gitea-runners/opentofu apply .sisyphus/evidence/task-12-destroy.plan
+```
+
+## Partial OpenTofu apply recovery
+
+If `tofu apply` fails after creating some resources, do not destroy blindly.
+First reconcile state and inspect what the stack thinks exists:
+
+```sh
+tofu -chdir=infra/gitea-runners/opentofu plan -refresh-only -out=.sisyphus/evidence/task-12-refresh.plan
+tofu -chdir=infra/gitea-runners/opentofu show -no-color .sisyphus/evidence/task-12-refresh.plan
+tofu -chdir=infra/gitea-runners/opentofu state list
+```
+
+Then rerun the normal plan path. Use `-target` only as a last resort when a
+single resource is stuck and the drift is understood.
+
+## S3 backend recovery
+
+If backend init or state access fails, first verify the bucket and versioning
+outside OpenTofu, then reconfigure the backend:
+
+```sh
+nix run nixpkgs#awscli2 -- s3api head-bucket --bucket gitea-runner-hectic-lab
+nix run nixpkgs#awscli2 -- s3api get-bucket-versioning --bucket gitea-runner-hectic-lab
+tofu -chdir=infra/gitea-runners/opentofu init -reconfigure
+tofu -chdir=infra/gitea-runners/opentofu plan
+```
+
+If the backend reports a stale lock, confirm no `tofu` process is active, then
+use `tofu force-unlock <LOCK_ID>` with the lock id from the error. Never force
+unlock a live plan or apply.
+
+## Gitea outage troubleshooting
+
+Use the public HTTPS service, not `localhost` or `127.0.0.1` inside job
+containers.
+
+```sh
+kubectl -n gitea-runners run gitea-outage-probe --rm --restart=Never --image=curlimages/curl:8.10.1 -- curl -fsS https://gitea.hectic-lab.com/api/healthz
+kubectl -n gitea-runners logs statefulset/gitea-runner -c runner --tail=200 | grep -E 'connection refused|timeout|tls|certificate|temporary failure' || true
+kubectl -n gitea-runners get events --sort-by=.lastTimestamp | tail -n 50
+```
+
+If Gitea is down, keep the existing StatefulSet and PVCs intact. Do not delete
+`/data/.runner` just because the service is unavailable. Once Gitea returns,
+repeat the token rotation or re-registration path if a pod restarted while the
+service was unavailable and lost its runner identity.
+
+## Release checklist
+
+Do not release unless the following evidence files exist and are readable:
+
+- `.sisyphus/evidence/task-5-cluster-plan.txt`
+- `.sisyphus/evidence/task-5-secret-plan-scan.txt`
+- `.sisyphus/evidence/task-9-deploy.txt`
+- `.sisyphus/evidence/task-9-secret-mount.txt`
+- `.sisyphus/evidence/task-10-ubuntu-workflow.txt`
+- `.sisyphus/evidence/task-10-nix-workflow.txt`
+- `.sisyphus/evidence/task-11-scale.txt`
+- `.sisyphus/evidence/task-11-restart-cleanup.txt`
+
+If any evidence file is missing, stop and collect it before treating the runbook
+as complete.

legacy/default.nix

@@ -0,0 +1,8 @@
+{ pkgs, ... }:
+let 
+  writers = pkgs.callPackage ./writer  { };
+in {
+  helpers = pkgs.callPackage ./helper { };
+  # NOTE(yukkop): duplicate writers in root of legacyPackages and writers due nixpkgs legacyPackages consistency
+  writers = writers;
+} // writers

legacy/helper/default.nix

@@ -0,0 +1,4 @@
+{ callPackage }: {
+  posix-shell = callPackage ./posix-shell {};
+  steam       = callPackage ./steam       {};
+}

legacy/helper/posix-shell/change_namespace.sh

@@ -0,0 +1,41 @@
+: "${OLD_NAMESPACE:=}"
+
+nl=$(printf '\nx')
+nl=${nl%x}
+
+___pop_namespace() {
+    v=${OLD_NAMESPACE%%"$nl"*}
+
+    case $OLD_NAMESPACE in
+        *"$nl"*)
+            OLD_NAMESPACE=${OLD_NAMESPACE#*"$nl"}
+            ;;
+        *)
+            OLD_NAMESPACE=
+            ;;
+    esac
+
+    printf '%s\n' "$v"
+}
+
+___peek_namespace() {
+    printf '%s\n' "${OLD_NAMESPACE%%"$nl"*}"
+}
+
+___push_namespace() {
+    if [ -n "$OLD_NAMESPACE" ]; then
+        OLD_NAMESPACE=$1"$nl$OLD_NAMESPACE"
+    else
+        OLD_NAMESPACE=$1
+    fi
+}
+
+change_namespace() {
+  ___push_namespace "$HECTIC_NAMESPACE"
+  export HECTIC_NAMESPACE="$1"
+}
+
+restore_namespace() {
+  HECTIC_NAMESPACE=$(___pop_namespace)
+  export HECTIC_NAMESPACE
+}

legacy/helper/posix-shell/colors.sh

@@ -0,0 +1,54 @@
+NC='\033[0m'
+
+# Regular text colors
+BLACK='\033[30m'
+RED='\033[31m'
+GREEN='\033[32m'
+YELLOW='\033[33m'
+BLUE='\033[34m'
+MAGENTA='\033[35m'
+CYAN='\033[36m'
+WHITE='\033[37m'
+
+# Bright text colors
+BBLACK='\033[90m'
+BRED='\033[91m'
+BGREEN='\033[92m'
+BYELLOW='\033[93m'
+BBLUE='\033[94m'
+BMAGENTA='\033[95m'
+BCYAN='\033[96m'
+BWHITE='\033[97m'
+
+# Background colors
+BG_BLACK='\033[40m'
+BG_RED='\033[41m'
+BG_GREEN='\033[42m'
+BG_YELLOW='\033[43m'
+BG_BLUE='\033[44m'
+BG_MAGENTA='\033[45m'
+BG_CYAN='\033[46m'
+BG_WHITE='\033[47m'
+
+# Bright background colors
+BG_BBLACK='\033[100m'
+BG_BRED='\033[101m'
+BG_BGREEN='\033[102m'
+BG_BYELLOW='\033[103m'
+BG_BBLUE='\033[104m'
+BG_BMAGENTA='\033[105m'
+BG_BCYAN='\033[106m'
+BG_BWHITE='\033[107m'
+
+# Text effects
+RESET='\033[0m'
+BOLD='\033[1m'
+DIM='\033[2m'
+ITALIC='\033[3m'
+UNDERLINE='\033[4m'
+BLINK='\033[5m'
+INVERSE='\033[7m'
+HIDDEN='\033[8m'
+STRIKE='\033[9m'
+
+: "$NC" "$BLACK" "$RED" "$GREEN" "$YELLOW" "$BLUE" "$MAGENTA" "$CYAN" "$WHITE" "$BBLACK" "$BRED" "$BGREEN" "$BYELLOW" "$BBLUE" "$BMAGENTA" "$BCYAN" "$BWHITE" "$BG_BLACK" "$BG_RED" "$BG_GREEN" "$BG_YELLOW" "$BG_BLUE" "$BG_MAGENTA" "$BG_CYAN" "$BG_WHITE" "$BG_BBLACK" "$BG_BRED" "$BG_BGREEN" "$BG_BYELLOW" "$BG_BBLUE" "$BG_BMAGENTA" "$BG_BCYAN" "$BG_BWHITE" "$RESET" "$BOLD" "$DIM" "$ITALIC" "$UNDERLINE" "$BLINK" "$INVERSE" "$HIDDEN" "$STRIKE"

legacy/helper/posix-shell/default.nix

@@ -0,0 +1,27 @@
+{ dash, hectic }: let
+  shell = "${dash}/bin/dash";
+  bashOptions = [
+    "errexit"
+    "nounset"
+  ];
+in {
+  log = hectic.writeDash "log.sh" ''
+    ${builtins.readFile ./colors.sh}
+    ${builtins.readFile ./log.sh}
+  '';
+  colors = hectic.writeDash "colors.sh" ''
+    ${builtins.readFile ./colors.sh}
+  '';
+  change_namespace = hectic.writeDash "change_namespace.sh" ''
+    ${builtins.readFile ./change_namespace.sh}
+  '';
+  quote = hectic.writeDash "quote.sh" ''
+    ${builtins.readFile ./quote.sh}
+  '';
+  pager_or_cat = hectic.writeDash "pager_or_cat.sh" ''
+    ${builtins.readFile ./pager_or_cat.sh}
+  '';
+  with_closed_fds = hectic.writeDash "with_closed_fds.sh" ''
+    ${builtins.readFile ./with_closed_fds.sh}
+  '';
+}

legacy/helper/posix-shell/log.sh

@@ -0,0 +1,155 @@
+#!/bin/dash
+
+# Hectic shell logger
+#
+# Usage:
+#   # Including
+#   . <this file>
+#
+#   # Required 
+#   colors.sh
+#
+#   # In your script (recommended: do NOT export HECTIC_NAMESPACE)
+#   HECTIC_NAMESPACE="my-script"   # optional, defaults to basename "$0"
+#   #   # Then use:
+#   log info   'starting up'
+#   log debug  "value=${val}"
+#   log error  "failed: ${WHITE}${reason}${NC} red text again"
+#
+#   # Note:
+#   When you use NC to reset terminal colors inside log output,
+#   it resets back to the log level’s color instead of the terminal default.
+
+: "${HECTIC_NAMESPACE="$(basename "$0")"}"
+: "${HECTIC_LOG:=trace}"  # e.g. "info;ns1=debug;ns2=trace"
+
+validate_log_level_spec() {
+    spec=$HECTIC_LOG
+
+    levels="trace debug info notice warn error"
+
+    ok_level() {
+        for l in $levels; do
+            [ "$l" = "$1" ] && return 0
+        done
+        return 1
+    }
+
+    oldIFS=$IFS
+    IFS=';'
+    # shellcheck disable=SC2086
+    set -- $spec
+    IFS=$oldIFS
+
+    for tok; do
+        case $tok in
+            *=*)
+                ns=${tok%%=*}
+                lvl=${tok#*=}
+                [ -n "$ns" ] || return 1
+                ok_level "$lvl" || return 1
+                ;;
+            *)
+                ok_level "$tok" || return 1
+                ;;
+        esac
+    done
+
+    return 0
+}
+
+exec 3>&2
+trap 'exec 3>&-' EXIT INT HUP
+
+validate_log_level_spec || { printf "%b%b\n" "${BBLACK}${HECTIC_NAMESPACE}> " "${color}invalid HECTIC_LOG syntax${NC}" "$@" >&3; exit 1; }
+
+log_level_num() {
+    case $1 in
+        trace)        printf %s 0 ;;
+        debug)        printf %s 1 ;;
+        info)         printf %s 2 ;;
+        notice)       printf %s 3 ;;
+        warn)         printf %s 4 ;;
+        error|panic)  printf %s 5 ;;
+        *)            printf %s 2 ;; # default info
+    esac
+}
+
+
+log_effective_level() {
+    spec=$HECTIC_LOG
+    ns=$HECTIC_NAMESPACE
+
+    default_level=
+    ns_level=
+
+    oldIFS=$IFS
+    IFS=';'
+    # shellcheck disable=SC2086
+    set -- $spec
+    IFS=$oldIFS
+
+    for tok; do
+        case $tok in
+            *=*)
+                name=${tok%%=*}
+                lvl=${tok#*=}
+                [ "$name" = "$ns" ] && ns_level=$lvl
+                ;;
+            *)
+                [ -z "$default_level" ] && default_level=$tok
+                ;;
+        esac
+    done
+
+    printf '%s\n' "${ns_level:-${default_level:-info}}"
+}
+
+log_allowed() {
+    msg_level="${1:?}"
+    eff_level="$(log_effective_level)"
+
+    msg_n="$(log_level_num "$msg_level")"
+    eff_n="$(log_level_num "$eff_level")"
+
+    [ "$msg_n" -ge "$eff_n" ]
+}
+
+# log(level, text...)
+log() {
+    delimetr=${DELIMETR:-' '};
+    level="${1:?}"
+    log_allowed "$level" || return 0
+
+    case "$level" in
+        trace)         color="$MAGENTA"  ;;
+        debug)         color="$BLUE"     ;;
+        info)          color="$GREEN"    ;;
+        notice)        color="$CYAN"     ;;
+        warn)          color="$YELLOW"   ;;
+        error)         color="$RED"      ;;
+        panic)         color="$BRED"     ;;
+        *)       
+    color="$WHITE"
+    NO_SHIFT=1
+  ;;
+    esac
+
+    [ ${NO_SHIFT+x} ] || shift
+
+    # shellcheck disable=SC2059
+    # shellcheck disable=SC2046
+    [ "$level" = panic ] && printf "${BBLACK}${HECTIC_NAMESPACE}> $BRED%b$NC\n" \
+      '' \
+      '' \
+      'this panic is unexpected behavior of program and/or bug' \
+      'please contact the developer' \
+      '' \
+      ''
+
+    # shellcheck disable=SC1003
+    fmt="$(printf "%s$delimetr" "$@" | sed 's/\\033\[0m/''\'"$color"'/g')"
+    shift
+    # shellcheck disable=SC1003
+    printf "${BBLACK}${HECTIC_NAMESPACE}> %b\n" "$color$fmt$NC" >&3
+}

legacy/helper/posix-shell/pager_or_cat.sh

@@ -0,0 +1,8 @@
+pager_or_cat_init() {
+  # Pipe to pager only if stdout is a terminal, otherwise output directly
+  if [ -t 1 ]; then
+    PAGER_OR_CAT="${PAGER:-less}"
+  else
+    PAGER_OR_CAT=cat
+  fi
+}

legacy/helper/posix-shell/quote.sh

@@ -0,0 +1 @@
+quote() { printf "'%s'" "$(printf %s "$1" | sed "s/'/'\\\\''/g")"; }

legacy/helper/posix-shell/with_closed_fds.sh

@@ -0,0 +1,22 @@
+#!/bin/dash
+
+# with_closed_fds -- run command with leaked file descriptors closed
+#
+# Shell libraries (e.g. hectic logger) may open extra file descriptors
+# (like fd 3 as a dup of stderr). Child processes inherit these fds.
+# Long-running daemons (postgres, postgrest) that keep fd 3 open can
+# prevent the terminal from returning to the prompt even after the
+# spawning script exits.
+#
+# Usage:
+#   with_closed_fds pg_ctl -D "$data" -w start
+#   with_closed_fds postgrest "$config" > "$log" 2>&1 &
+#
+# Runs the command in a subshell where fds 3-9 are redirected to
+# /dev/null. The parent shell's fd table is untouched.
+with_closed_fds() {
+    (
+        exec 3>/dev/null 4>/dev/null 5>/dev/null 6>/dev/null 7>/dev/null 8>/dev/null 9>/dev/null
+        "$@"
+    )
+}

legacy/helper/steam/default.nix

@@ -0,0 +1,27 @@
+{ stdenv, steamcmd }: {
+  buildSteamServer = steamId: stdenv.mkDerivation {
+    pname = "astroneer-dedicated-server";
+    version = "latest";
+  
+    src = null;
+  
+    nativeBuildInputs = [
+      steamcmd
+    ];
+  
+    buildPhase = ''
+      export HOME=$TMPDIR
+      mkdir -p $out
+      steamcmd \
+        +force_install_dir $out \
+        +login anonymous \
+        +app_update ${steamId} validate \
+        +quit
+    '';
+  
+    installPhase = "true";
+  
+    dontFixup = true;
+    dontStrip = true;
+  };
+}

legacy/writer/default.nix

@@ -0,0 +1,15 @@
+{ callPackage }: rec {
+  writeShellApplication = callPackage ./writeShellApplication.nix  {};
+  writeDash             = callPackage ./writeDash.nix              {};
+  writeC                = callPackage ./writeC.nix                 {};
+  writeCBin             = name: writeC "/bin/${name}";
+  writeMinCBin          = name: includes: body: writeMinC "/bin/${name}" includes body;
+  writeMinC             = name: includes: body:
+        writeC name ''
+          ${builtins.concatStringsSep "\n" (map (h: "#include " + h) includes)}
+
+          int main(int argc, char *argv[]) {
+              ${body}
+          }
+        '';
+}

legacy/writer/writeC.nix

@@ -0,0 +1,35 @@
+{ lib, writers, gcc }:
+name: argsOrScript:
+  if 
+    lib.isAttrs argsOrScript 
+    && !lib.isDerivation argsOrScript
+  then
+    writers.makeBinWriter (
+      argsOrScript
+      // {
+        compileScript = ''
+          # Force gcc to treat the input file as C code
+          ${gcc}/bin/gcc -fsyntax-only -xc $contentPath
+          if [ $? -ne 0 ]; then
+            echo "Syntax check failed"
+            exit 1
+          fi
+          ${gcc}/bin/gcc -xc -o $out $contentPath
+        '';
+      }
+    )
+    name
+  else
+    writers.makeBinWriter {
+      compileScript = ''
+        # Force gcc to treat the input file as C code
+        ${gcc}/bin/gcc -fsyntax-only -xc $contentPath
+        if [ $? -ne 0 ]; then
+          echo "Syntax check failed"
+          exit 1
+        fi
+        ${gcc}/bin/gcc -xc -o $out $contentPath
+      '';
+    }
+    name
+    argsOrScript

legacy/writer/writeDash.nix

@@ -0,0 +1,5 @@
+  { dash, lib, writers }: name: argsOrScript:
+  if lib.isAttrs argsOrScript && !lib.isDerivation argsOrScript then
+    writers.makeScriptWriter (argsOrScript // { interpreter = "${lib.getExe dash}"; }) name
+  else
+    writers.makeScriptWriter { interpreter = "${lib.getExe dash}"; } name argsOrScript

legacy/writer/writeShellApplication.nix

@@ -0,0 +1,157 @@
+{
+  writeTextFile,
+  lib,
+  shellcheck-minimal,
+  stdenv,
+  runtimeShell,
+}:
+{
+  /*
+     The name of the script to write.
+
+     Type: String
+  */
+  name,
+  /*
+     The shell script's text, not including a shebang.
+
+     Type: String
+  */
+  text,
+  /*
+     Inputs to add to the shell script's `$PATH` at runtime.
+
+     Type: [String|Derivation]
+  */
+  runtimeInputs ? [ ],
+  /*
+     Extra environment variables to set at runtime.
+
+     Type: AttrSet
+  */
+  runtimeEnv ? null,
+  /*
+     `stdenv.mkDerivation`'s `meta` argument.
+
+     Type: AttrSet
+  */
+  meta ? { },
+  /*
+     `stdenv.mkDerivation`'s `passthru` argument.
+
+     Type: AttrSet
+  */
+  passthru ? { },
+  /*
+     The `checkPhase` to run. Defaults to `shellcheck` on supported
+     platforms and `bash -n`.
+
+     The script path will be given as `$target` in the `checkPhase`.
+
+     Type: String
+  */
+  checkPhase ? null,
+  /*
+     Checks to exclude when running `shellcheck`, e.g. `[ "SC2016" ]`.
+
+     See <https://www.shellcheck.net/wiki/> for a list of checks.
+
+     Type: [String]
+  */
+  excludeShellChecks ? [ ],
+  /*
+     Extra command-line flags to pass to ShellCheck.
+
+     Type: [String]
+  */
+  extraShellCheckFlags ? [ ],
+  /*
+     Bash options to activate with `set -o` at the start of the script.
+
+     Defaults to `[ "errexit" "nounset" "pipefail" ]`.
+
+     Type: [String]
+  */
+  bashOptions ? [
+    "errexit"
+    "nounset"
+    "pipefail"
+  ],
+  /*
+    Extra arguments to pass to `stdenv.mkDerivation`.
+
+    :::{.caution}
+    Certain derivation attributes are used internally,
+    overriding those could cause problems.
+    :::
+
+    Type: AttrSet
+  */
+  derivationArgs ? { },
+  /*
+     Whether to inherit the current `$PATH` in the script.
+
+     Type: Bool
+  */
+  inheritPath ? true,
+
+  shell ? runtimeShell,
+}:
+writeTextFile {
+  inherit
+    name
+    meta
+    passthru
+    derivationArgs
+    ;
+  executable = true;
+  destination = "/bin/${name}";
+  allowSubstitutes = true;
+  preferLocalBuild = false;
+  text = ''
+    #!${shell}
+    ${lib.concatMapStringsSep "\n" (option: "set -o ${option}") bashOptions}
+  ''
+  + lib.optionalString (runtimeEnv != null) (
+    lib.concatStrings (
+      lib.mapAttrsToList (name: value: ''
+        ${lib.toShellVar name value}
+        export ${name}
+      '') runtimeEnv
+    )
+  )
+  + lib.optionalString (runtimeInputs != [ ]) ''
+
+    export PATH="${lib.makeBinPath runtimeInputs}${lib.optionalString inheritPath ":$PATH"}"
+  ''
+  + ''
+
+    ${text}
+  '';
+
+  checkPhase =
+    let
+      excludeFlags = lib.optionals (excludeShellChecks != [ ]) [
+        "--exclude"
+        (lib.concatStringsSep "," excludeShellChecks)
+      ];
+      # GHC (=> shellcheck) isn't supported on some platforms (such as risc-v)
+      # but we still want to use writeShellApplication on those platforms
+      shellcheckCommand = lib.optionalString shellcheck-minimal.compiler.bootstrapAvailable ''
+        # use shellcheck which does not include docs
+        # pandoc takes long to build and documentation isn't needed for just running the cli
+        ${lib.getExe shellcheck-minimal} ${
+          lib.escapeShellArgs (excludeFlags ++ extraShellCheckFlags)
+        } "$target"
+      '';
+    in
+    if checkPhase == null then
+      ''
+        runHook preCheck
+        ${stdenv.shellDryRun} "$target"
+        ${shellcheckCommand}
+        runHook postCheck
+      ''
+    else
+      checkPhase;
+}

lib/default.nix

@@ -0,0 +1,269 @@
+{ flake, inputs, self }: let
+  nixpkgs = inputs.nixpkgs;
+  lib = nixpkgs.lib;
+  recursiveUpdate = nixpkgs.lib.recursiveUpdate;
+
+  envErrorMessage = varName: "Error: The ${varName} environment variable is not set.";
+
+  AllSystems = [
+    "aarch64-darwin"
+    "aarch64-linux"    
+    "armv5tel-linux"     
+    "armv6l-linux"    
+    "armv7l-linux"      
+    "i686-linux"
+    "mipsel-linux"
+    "powerpc64le-linux"
+    "riscv64-linux"
+    "x86_64-darwin"
+    "x86_64-linux"
+  ];
+
+  commonSystems = [
+    "x86_64-linux"
+    "aarch64-linux"
+    "x86_64-darwin"
+    "aarch64-darwin"
+  ];
+
+  cudaUnfreeNames = [
+    "cuda_nvcc"
+    "cuda_cudart"
+    "cuda_cuobjdump"
+    "cuda_cupti"
+    "cuda_nvdisasm"
+    "cuda_cccl"
+    "cuda_nvml_dev"
+    "cuda_nvrtc"
+    "cuda_nvtx"
+    "cuda_profiler_api"
+
+    "libcusparse_lt"
+    "libcublas"
+    "libcufft"
+    "libcufile"
+    "libcurand"
+    "libcusolver"
+    "libnvjitlink"
+    "libcusparse"
+    "cudnn"
+  ];
+
+  cudaUnfreePredicate = pkg:
+    builtins.elem (nixpkgs.lib.getName pkg) cudaUnfreeNames;
+
+  forSystemsWithPkgs = supportedSystems: pkgOverlays: f:
+    builtins.foldl' (
+      acc: system: let
+        pkgs = import nixpkgs {
+          inherit system;
+          overlays = pkgOverlays;
+          config.allowUnfreePredicate = cudaUnfreePredicate;
+        };
+        systemOutputs = f {
+          system = system;
+          pkgs = pkgs;
+        };
+      in
+        recursiveUpdate acc systemOutputs
+    ) {}
+    supportedSystems;
+
+  forAllSystemsWithPkgs = pkgOverlays: f: forSystemsWithPkgs AllSystems pkgOverlays f;
+
+  parseEnv = import ./parse-env.nix;
+
+  dotEnv = builtins.getEnv "DOTENV";
+  minorEnvironment =
+    if dotEnv != ""
+    then
+      if builtins.pathExists dotEnv
+      then parseEnv dotEnv
+      else throw "${dotEnv} file not exist"
+    else if builtins.pathExists ./.env
+    then parseEnv ./.env
+    else {};
+in {
+  # -- For all systems --
+  inherit dotEnv minorEnvironment parseEnv forAllSystemsWithPkgs forSystemsWithPkgs commonSystems AllSystems cudaUnfreeNames cudaUnfreePredicate;
+
+  forSystems = systems: nixpkgs.lib.genAttrs systems;
+  forAllSystems = nixpkgs.lib.genAttrs AllSystems;
+
+  shellModules = {
+    logs = builtins.readFile ./shell/logs.sh;
+    check-tool = builtins.readFile ./shell/check-tool.sh;
+    local-dir = builtins.readFile ./shell/local-dir.sh;
+    load-sops = builtins.readFile ./shell/load-sops.sh;
+  };
+
+  sharedShellAliases = {
+    jc = ''journalctl'';
+    sc = ''journalctl'';
+    nv = ''nvim'';
+  };
+
+  sharedShellAliasesForDevVm = self.lib.sharedShellAliases // {
+    sd = "shutdown now";
+  };
+
+  readEnvironment = { envVarsToRead, prefix ? "" }:
+    builtins.listToAttrs
+    (map (name: {
+        inherit name;
+        value = self.lib.getEnv "${prefix}${name}";
+      })
+      envVarsToRead);
+
+  # -- Env processing --
+  getEnv = varName: let
+    var = builtins.getEnv varName;
+  in
+    if var != ""
+    then var
+    else if minorEnvironment ? varName
+    then minorEnvironment."${varName}"
+    else throw (envErrorMessage varName);
+
+  # -- Cargo.toml --
+  cargoToml = src: (builtins.fromTOML (builtins.readFile "${src}/Cargo.toml"));
+
+  # Consolidated SQL bundles for the `hectic` schema. Single source of truth
+  # for everything that creates objects in the `hectic` namespace, used by
+  # migrator (init-time), db-dev/database hydrate, and db-ops secrets loading. Consumers apply
+  # the full bundle via lib/hook/apply-hectic-bundle.sh.
+  #
+  # The whole hectic system shares one `versionString`; `hectic-version.sql`
+  # registers (`'hectic'`, versionString) into `hectic.version` and raises an
+  # exception on mismatch. Per-hook version rows are intentionally absent.
+  #
+  # Each entry exposes:
+  #   * .sql   — file contents as a string, with @HECTIC_VERSION@ substituted
+  #   * .path  — Nix store path (only on entries that need no substitution)
+  hectic = let
+    versionString = lib.fileContents ./hook/sql/HECTIC_VERSION;
+    static = path: { inherit path; sql = builtins.readFile path; };
+    templated = path: let
+      sql = builtins.replaceStrings
+        [ "@HECTIC_VERSION@" ]
+        [ versionString ]
+        (builtins.readFile path);
+    in {
+      inherit sql;
+      path = builtins.toFile (builtins.baseNameOf (toString path)) sql;
+    };
+  in rec {
+    inherit versionString;
+    version     = templated ./hook/sql/hectic-version.sql;
+    secret      = static    ./hook/sql/hectic-secret.sql;
+    migration   = static    ./hook/sql/hectic-migration.sql;
+    inheritance = static    ./hook/sql/hectic-inheritance.sql;
+    bundleFiles = [
+      version.path
+      secret.path
+      migration.path
+      inheritance.path
+    ];
+    applyBundleScript =
+      builtins.replaceStrings
+        [
+          "@HECTIC_VERSION_SQL@"
+          "@HECTIC_SECRET_SQL@"
+          "@HECTIC_MIGRATION_SQL@"
+          "@HECTIC_INHERITANCE_SQL@"
+        ]
+        [
+          "${version.path}"
+          "${secret.path}"
+          "${migration.path}"
+          "${inheritance.path}"
+        ]
+        (builtins.readFile ./hook/apply-hectic-bundle.sh);
+  };
+
+  # Back-compat alias. Prefer `self.lib.hectic.inheritance`.
+  hecticInheritance = let
+    path = ./hook/sql/hectic-inheritance.sql;
+  in {
+    inherit path;
+    sql = builtins.readFile path;
+  };
+
+  ssh.keys = {
+    hetzner-test = {
+      yukkop = ''ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIJ8scy1tv6zfXX6xyaukhO/fsZwif5rC89DvXNc6XxOf'';
+    };
+  };
+
+  readPackages = callPackage: path: extraArgs:
+  with lib;
+  with builtins;
+  pipe path [
+    readDir
+    (filterAttrs (_: type: type == "directory"))
+    (filterAttrs (name: _: pathExists "${path}/${name}/default.nix"))
+    (mapAttrs (name: _: callPackage "${path}/${name}" extraArgs))
+  ];
+
+  # Like readModulesRecursive, but reads module structure as a one-level keys,
+  # so that it is suited for `nix flake show`
+  # ```nix
+  # {
+  #   "foo.bar" = import ./module/foo/bar.nix
+  # }
+  # ```
+  readModulesRecursive' = path: extraArgs:
+    with lib;
+    with builtins; let
+      collectPaths = dir: prefix:
+        concatLists (mapAttrsToList (name: type: let
+          path' = dir + "/${name}";
+          name' = if prefix == "" then name else "${prefix}/${name}";
+        in
+          if type == "directory"
+          then collectPaths path' name'
+          else [{
+            inherit path';
+            name = name';
+          }]
+        ) (readDir dir));
+      paths = filter (path': hasSuffix ".nix" path'.name) (collectPaths path "");
+      pathToName = flip pipe [
+        (replaceStrings ["/" ".nix"] ["." ""])
+        (removeSuffix ".nix")
+      ];
+      attrList =
+        map (path': {
+          name = pathToName path'.name;
+          value = import path'.path' extraArgs;
+        })
+        paths;
+    in
+      listToAttrs attrList;
+
+  nixpkgs-lib = nixpkgs.lib;
+} // rec {
+  /* Supplied a directory, reads it's recursive structure into NixOS modules, so
+     that provided a `./module` dir with `module/foo/bar.nix` in it it outputs
+     ```nix
+     {
+       foo.bar = import ./module/foo/bar.nix
+     }
+    ```
+  */
+  readModulesRecursive = path:
+    lib.mapAttrs' (
+      name: value: let
+        name' = builtins.replaceStrings [".nix"] [""] name;
+      in
+        if value == "regular"
+        then {
+          name = name';
+          value = import "${path}/${name}";
+        }
+        else {
+          inherit name;
+          value = readModulesRecursive "${path}/${name}";
+        }
+    ) (builtins.readDir path);
+}

lib/hook/apply-hectic-bundle.sh

@@ -0,0 +1,51 @@
+#!/bin/dash
+# Applies the full hectic SQL bundle to a PostgreSQL database, in order:
+#   1. version    (hard-fails on version mismatch)
+#   2. secret     (hectic.secret table + load_secrets_from_env + get_secret)
+#   3. migration  (hectic.migration table + domains + sha256_lower trigger)
+#   4. inheritance (created_at/updated_at/immutable enforcement triggers)
+#
+# Idempotent: each SQL file uses IF NOT EXISTS / CREATE OR REPLACE.
+#
+# Usage:
+#   apply_hectic_bundle <PGURL> [<DOTENV_CONTENT>]
+#
+# If DOTENV_CONTENT is non-empty, it is base64-encoded and then loaded into
+# hectic.secret via hectic.load_secrets_from_env() after the bundle is applied.
+# SQL file paths are substituted by Nix evaluation time.
+
+apply_hectic_bundle() {
+  pgurl="${1:-}"
+  env_content="${2:-}"
+
+  if [ -z "$pgurl" ]; then
+    printf '%s\n' 'apply-hectic-bundle: PGURL is required (arg 1)' >&2
+    return 3
+  fi
+
+  set -- \
+    "@HECTIC_VERSION_SQL@" \
+    "@HECTIC_SECRET_SQL@" \
+    "@HECTIC_MIGRATION_SQL@" \
+    "@HECTIC_INHERITANCE_SQL@"
+
+  for sql_path do
+    if [ ! -r "$sql_path" ]; then
+      printf '%s\n' "apply-hectic-bundle: SQL file not readable: $sql_path" >&2
+      return 1
+    fi
+  done
+
+  for sql_path do
+    psql "$pgurl" -v ON_ERROR_STOP=1 -f "$sql_path" || return 1
+  done
+
+  if [ -n "$env_content" ]; then
+    env_content_b64="$(printf '%s' "$env_content" | base64 | tr -d '\n')" || return 1
+    psql "$pgurl" -v ON_ERROR_STOP=1 <<SQL || return 1
+SELECT hectic.load_secrets_from_env(convert_from(decode('$env_content_b64', 'base64'), 'UTF8'));
+SQL
+  fi
+
+  return 0
+}

lib/hook/sql/HECTIC_VERSION

@@ -0,0 +1 @@
+0.1.0

lib/hook/sql/README.md

@@ -0,0 +1,99 @@
+# hectic SQL bundle
+
+Single source of truth for every object created in the `hectic` PostgreSQL
+schema. Consumed by:
+
+- `package/migrator` — applies the bundle on `migrator init` (mandatory).
+- `package/db-tool` — applies the bundle in `db-dev` / `database hydrate`
+  (default; opt out with `--no-hook`) and in `db-ops secrets load`.
+- External consumers (e.g. `proxydoe`) — invoke `psql -f` directly against the
+  paths exposed via `self.lib.hectic.*.path`.
+
+## Layout
+
+| File | Purpose |
+| --- | --- |
+| `HECTIC_VERSION` | Single version string for the whole bundle (e.g. `0.1.0`). Read via `lib.fileContents`. |
+| `hectic-version.sql` | Templated. Creates `hectic.version`, inserts the current `versionString`, raises on mismatch. |
+| `hectic-secret.sql` | Creates `hectic.secret`, `hectic.load_secrets_from_env(text)`, `hectic.get_secret(text)`. |
+| `hectic-migration.sql` | Creates the `hectic.migration` table and supporting domains/triggers used by `migrator`. |
+| `hectic-inheritance.sql` | Creates `hectic.created_at`, `hectic.updated_at`, `hectic.immutable` parent tables and the DDL event triggers that enforce inheritance, attach `BEFORE UPDATE` triggers, and block DML on immutable tables outside `migration_mode`. |
+
+`hectic-version.sql` is templated at Nix evaluation time: `@HECTIC_VERSION@`
+is substituted with the contents of `HECTIC_VERSION`. All other files are
+applied verbatim.
+
+## Apply order
+
+The bundle MUST be applied in this order (enforced by
+`apply-hectic-bundle.sh`):
+
+1. `hectic-version.sql` — version check first; aborts the rest on mismatch.
+2. `hectic-secret.sql`
+3. `hectic-migration.sql`
+4. `hectic-inheritance.sql`
+
+Re-applying the bundle is idempotent — every CREATE uses
+`IF NOT EXISTS` / `CREATE OR REPLACE`, and the version check accepts a row
+that already matches.
+
+## Nix API (`self.lib.hectic`)
+
+```nix
+self.lib.hectic = {
+  versionString;          # e.g. "0.1.0"
+  version     = { sql; path; };     # templated
+  secret      = { sql; path; };
+  migration   = { sql; path; };
+  inheritance = { sql; path; };
+  bundleFiles;                      # ordered bundle file paths
+  applyBundleScript;                # generated helper shell source with paths embedded
+};
+```
+
+`.sql` is the file contents as a string. `.path` is the Nix store path of the
+materialized file to pass to `psql -f`. `version.path` is generated at Nix
+evaluation time from the templated SQL; the other `*.path` entries point at the
+verbatim source files in the store.
+
+## Shell helper (`apply-hectic-bundle.sh`)
+
+`lib/hook/apply-hectic-bundle.sh` is a dash-compatible helper template.
+`self.lib.hectic.applyBundleScript` is the generated shell source with concrete
+SQL paths embedded at Nix evaluation time. `migrator`, `db-dev`, and `db-ops` splice that
+shell source directly into their generated scripts. Public entry point:
+
+```sh
+apply_hectic_bundle <PGURL> [<DOTENV_CONTENT>]
+```
+
+- `<PGURL>` — full PostgreSQL connection string.
+- `<DOTENV_CONTENT>` — optional. When present, after applying the bundle the
+  helper invokes `hectic.load_secrets_from_env(<dotenv>)` inside a
+  dollar-quoted (`$ps_env$`) string so secret values cannot terminate the
+  literal.
+
+The SQL file paths are embedded into the helper at Nix evaluation time, so
+callers only need to source the generated script and call the function.
+External consumers that do not want to source the helper can still invoke
+`psql -f` against `self.lib.hectic.bundleFiles` or the individual
+`self.lib.hectic.*.path` entries directly.
+
+## Adding a new SQL file
+
+1. Add `lib/hook/sql/hectic-<name>.sql`.
+2. Wire it into `lib/default.nix` under `lib.hectic.<name>`.
+3. Add its `.path` to `lib.hectic.bundleFiles` in the correct order.
+4. Add a matching placeholder/replacement in `lib.hectic.applyBundleScript` and
+   update `lib/hook/apply-hectic-bundle.sh` to apply the file.
+5. Bump `HECTIC_VERSION` if the new content changes existing semantics.
+6. Update tests in `test/package/migrator/test/postgresql/init-hectic-bundle/`,
+   `test/package/db-tool/test/postgresql/hydrate-hook/`, and any `db-ops`
+   bundle-loading coverage.
+
+## Versioning
+
+`HECTIC_VERSION` is a single global version for the bundle, not per-file.
+Bump it on any breaking change to the schema. `hectic-version.sql` raises an
+exception when the database row diverges from the bundle version, forcing a
+deliberate migration before the rest of the bundle runs.

lib/hook/sql/hectic-inheritance.sql

@@ -0,0 +1,257 @@
+-- hectic.created_at / hectic.updated_at / hectic.immutable inheritance machinery.
+--
+-- Provides:
+--   * schema  hectic
+--   * tables  hectic.created_at(created_at TIMESTAMPTZ NOT NULL DEFAULT NOW())
+--             hectic.updated_at(updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW())
+--             hectic.immutable()                                  -- pure marker
+--   * function hectic.set_updated_at()  -- BEFORE UPDATE row trigger function
+--   * function hectic.block_immutable_dml()
+--         BEFORE INSERT/UPDATE/DELETE/TRUNCATE row+statement trigger function;
+--         allows DML iff current_setting('hectic.migration_mode', true) = 'on'.
+--   * GUC  hectic.inheritance_extra_excluded_schemas
+--         (text, comma-separated list of schemas the enforcement trigger skips)
+--   * GUC  hectic.migration_mode
+--         (text, 'on' enables DML on tables inheriting hectic.immutable.
+--          Intended use: SET LOCAL inside a migration transaction.)
+--   * event trigger hectic_enforce_created_at_inheritance
+--         RAISE EXCEPTION on CREATE TABLE that does not inherit hectic.created_at
+--   * event trigger hectic_attach_updated_at_trigger
+--         auto-attaches BEFORE UPDATE row trigger calling hectic.set_updated_at()
+--         on any new table that inherits hectic.updated_at and lacks one.
+--   * event trigger hectic_attach_immutable_triggers
+--         auto-attaches BEFORE INSERT/UPDATE/DELETE FOR EACH ROW and BEFORE
+--         TRUNCATE FOR EACH STATEMENT triggers calling hectic.block_immutable_dml()
+--         on any new table that inherits hectic.immutable and lacks them.
+--
+-- Idempotent: safe to run on an already-bootstrapped database.
+
+CREATE SCHEMA IF NOT EXISTS "hectic";
+
+CREATE TABLE IF NOT EXISTS "hectic"."created_at" (
+    "created_at" TIMESTAMPTZ NOT NULL DEFAULT NOW()
+);
+
+CREATE TABLE IF NOT EXISTS "hectic"."updated_at" (
+    "updated_at" TIMESTAMPTZ NOT NULL DEFAULT NOW()
+);
+
+CREATE TABLE IF NOT EXISTS "hectic"."immutable" ();
+
+DO $bootstrap$
+BEGIN
+  PERFORM set_config('hectic.inheritance_extra_excluded_schemas',
+                     current_setting('hectic.inheritance_extra_excluded_schemas', true),
+                     false);
+EXCEPTION WHEN undefined_object THEN
+  PERFORM set_config('hectic.inheritance_extra_excluded_schemas', '', false);
+END
+$bootstrap$;
+
+DO $bootstrap_mm$
+BEGIN
+  PERFORM set_config('hectic.migration_mode',
+                     current_setting('hectic.migration_mode', true),
+                     false);
+EXCEPTION WHEN undefined_object THEN
+  PERFORM set_config('hectic.migration_mode', '', false);
+END
+$bootstrap_mm$;
+
+CREATE OR REPLACE FUNCTION "hectic"."set_updated_at"() RETURNS trigger
+LANGUAGE plpgsql AS $fn$
+BEGIN
+  NEW."updated_at" := NOW();
+  RETURN NEW;
+END
+$fn$;
+
+CREATE OR REPLACE FUNCTION "hectic"."block_immutable_dml"() RETURNS trigger
+LANGUAGE plpgsql AS $fn$
+DECLARE
+  mm text;
+BEGIN
+  BEGIN
+    mm := current_setting('hectic.migration_mode', true);
+  EXCEPTION WHEN OTHERS THEN
+    mm := '';
+  END;
+  IF mm = 'on' THEN
+    IF TG_LEVEL = 'STATEMENT' THEN RETURN NULL; END IF;
+    IF TG_OP = 'DELETE' THEN RETURN OLD; END IF;
+    RETURN NEW;
+  END IF;
+  RAISE EXCEPTION
+    'hectic: table %.% inherits hectic.immutable; % blocked outside migration_mode',
+    quote_ident(TG_TABLE_SCHEMA), quote_ident(TG_TABLE_NAME), TG_OP
+    USING HINT = 'wrap the statement in a migration transaction with '
+              || 'SET LOCAL hectic.migration_mode = ''on''';
+END
+$fn$;
+
+CREATE OR REPLACE FUNCTION "hectic"."_is_excluded_schema"(p_schema text) RETURNS boolean
+LANGUAGE plpgsql STABLE AS $fn$
+DECLARE
+  extra text;
+  s     text;
+BEGIN
+  IF p_schema = 'hectic'
+     OR p_schema = 'information_schema'
+     OR p_schema LIKE 'pg\_%' ESCAPE '\'
+  THEN
+    RETURN true;
+  END IF;
+  BEGIN
+    extra := current_setting('hectic.inheritance_extra_excluded_schemas', true);
+  EXCEPTION WHEN OTHERS THEN
+    extra := '';
+  END;
+  IF extra IS NULL OR extra = '' THEN
+    RETURN false;
+  END IF;
+  FOREACH s IN ARRAY string_to_array(extra, ',') LOOP
+    IF btrim(s) = p_schema THEN
+      RETURN true;
+    END IF;
+  END LOOP;
+  RETURN false;
+END
+$fn$;
+
+CREATE OR REPLACE FUNCTION "hectic"."_table_inherits"(p_oid oid, p_parent regclass) RETURNS boolean
+LANGUAGE sql STABLE AS $fn$
+  SELECT EXISTS (
+    SELECT 1 FROM pg_inherits
+    WHERE inhrelid = p_oid AND inhparent = p_parent
+  );
+$fn$;
+
+CREATE OR REPLACE FUNCTION "hectic"."enforce_created_at_inheritance"() RETURNS event_trigger
+LANGUAGE plpgsql AS $fn$
+DECLARE
+  obj         record;
+  rel         pg_class;
+  schema_name text;
+  parent_oid  oid;
+BEGIN
+  parent_oid := 'hectic.created_at'::regclass;
+  FOR obj IN SELECT * FROM pg_event_trigger_ddl_commands() WHERE command_tag = 'CREATE TABLE'
+  LOOP
+    SELECT * INTO rel FROM pg_class WHERE oid = obj.objid;
+    IF NOT FOUND THEN CONTINUE; END IF;
+    IF rel.relpersistence = 't' THEN CONTINUE; END IF;
+    IF rel.relispartition THEN CONTINUE; END IF;
+    SELECT nspname INTO schema_name FROM pg_namespace WHERE oid = rel.relnamespace;
+    IF "hectic"."_is_excluded_schema"(schema_name) THEN CONTINUE; END IF;
+    IF NOT "hectic"."_table_inherits"(rel.oid, parent_oid) THEN
+      RAISE EXCEPTION
+        'hectic: table %.% must INHERITS (hectic.created_at)',
+        quote_ident(schema_name), quote_ident(rel.relname)
+        USING HINT = 'add INHERITS ("hectic"."created_at") to the CREATE TABLE statement, '
+                  || 'or add the schema to hectic.inheritance_extra_excluded_schemas';
+    END IF;
+  END LOOP;
+END
+$fn$;
+
+CREATE OR REPLACE FUNCTION "hectic"."attach_updated_at_trigger"() RETURNS event_trigger
+LANGUAGE plpgsql AS $fn$
+DECLARE
+  obj           record;
+  rel           pg_class;
+  schema_name   text;
+  parent_oid    oid;
+  trigger_name  text;
+  has_trigger   boolean;
+BEGIN
+  parent_oid := 'hectic.updated_at'::regclass;
+  FOR obj IN SELECT * FROM pg_event_trigger_ddl_commands() WHERE command_tag = 'CREATE TABLE'
+  LOOP
+    SELECT * INTO rel FROM pg_class WHERE oid = obj.objid;
+    IF NOT FOUND THEN CONTINUE; END IF;
+    IF rel.relpersistence = 't' THEN CONTINUE; END IF;
+    IF rel.relispartition THEN CONTINUE; END IF;
+    SELECT nspname INTO schema_name FROM pg_namespace WHERE oid = rel.relnamespace;
+    IF schema_name = 'hectic' THEN CONTINUE; END IF;
+    IF NOT "hectic"."_table_inherits"(rel.oid, parent_oid) THEN CONTINUE; END IF;
+    trigger_name := 'hectic_set_updated_at';
+    SELECT EXISTS (
+      SELECT 1 FROM pg_trigger
+      WHERE tgrelid = rel.oid AND tgname = trigger_name AND NOT tgisinternal
+    ) INTO has_trigger;
+    IF has_trigger THEN CONTINUE; END IF;
+    EXECUTE format(
+      'CREATE TRIGGER %I BEFORE UPDATE ON %I.%I FOR EACH ROW EXECUTE FUNCTION "hectic"."set_updated_at"()',
+      trigger_name, schema_name, rel.relname
+    );
+  END LOOP;
+END
+$fn$;
+
+DROP EVENT TRIGGER IF EXISTS "hectic_enforce_created_at_inheritance";
+CREATE EVENT TRIGGER "hectic_enforce_created_at_inheritance"
+  ON ddl_command_end
+  WHEN TAG IN ('CREATE TABLE')
+  EXECUTE FUNCTION "hectic"."enforce_created_at_inheritance"();
+
+DROP EVENT TRIGGER IF EXISTS "hectic_attach_updated_at_trigger";
+CREATE EVENT TRIGGER "hectic_attach_updated_at_trigger"
+  ON ddl_command_end
+  WHEN TAG IN ('CREATE TABLE')
+  EXECUTE FUNCTION "hectic"."attach_updated_at_trigger"();
+
+CREATE OR REPLACE FUNCTION "hectic"."attach_immutable_triggers"() RETURNS event_trigger
+LANGUAGE plpgsql AS $fn$
+DECLARE
+  obj         record;
+  rel         pg_class;
+  schema_name text;
+  parent_oid  oid;
+  has_row     boolean;
+  has_trunc   boolean;
+BEGIN
+  parent_oid := 'hectic.immutable'::regclass;
+  FOR obj IN SELECT * FROM pg_event_trigger_ddl_commands() WHERE command_tag = 'CREATE TABLE'
+  LOOP
+    SELECT * INTO rel FROM pg_class WHERE oid = obj.objid;
+    IF NOT FOUND THEN CONTINUE; END IF;
+    IF rel.relpersistence = 't' THEN CONTINUE; END IF;
+    IF rel.relispartition THEN CONTINUE; END IF;
+    SELECT nspname INTO schema_name FROM pg_namespace WHERE oid = rel.relnamespace;
+    IF schema_name = 'hectic' THEN CONTINUE; END IF;
+    IF NOT "hectic"."_table_inherits"(rel.oid, parent_oid) THEN CONTINUE; END IF;
+    SELECT EXISTS (
+      SELECT 1 FROM pg_trigger
+      WHERE tgrelid = rel.oid
+        AND tgname  = 'hectic_block_immutable_dml'
+        AND NOT tgisinternal
+    ) INTO has_row;
+    SELECT EXISTS (
+      SELECT 1 FROM pg_trigger
+      WHERE tgrelid = rel.oid
+        AND tgname  = 'hectic_block_immutable_truncate'
+        AND NOT tgisinternal
+    ) INTO has_trunc;
+    IF NOT has_row THEN
+      EXECUTE format(
+        'CREATE TRIGGER %I BEFORE INSERT OR UPDATE OR DELETE ON %I.%I '
+        || 'FOR EACH ROW EXECUTE FUNCTION "hectic"."block_immutable_dml"()',
+        'hectic_block_immutable_dml', schema_name, rel.relname
+      );
+    END IF;
+    IF NOT has_trunc THEN
+      EXECUTE format(
+        'CREATE TRIGGER %I BEFORE INSERT OR UPDATE OR DELETE OR TRUNCATE ON %I.%I '
+        || 'FOR EACH STATEMENT EXECUTE FUNCTION "hectic"."block_immutable_dml"()',
+        'hectic_block_immutable_truncate', schema_name, rel.relname
+      );
+    END IF;
+  END LOOP;
+END
+$fn$;
+
+DROP EVENT TRIGGER IF EXISTS "hectic_attach_immutable_triggers";
+CREATE EVENT TRIGGER "hectic_attach_immutable_triggers"
+  ON ddl_command_end
+  WHEN TAG IN ('CREATE TABLE')
+  EXECUTE FUNCTION "hectic"."attach_immutable_triggers"();

lib/hook/sql/hectic-migration.sql

@@ -0,0 +1,51 @@
+DO $bootstrap$
+BEGIN
+    IF NOT EXISTS (
+        SELECT 1
+        FROM pg_type t JOIN pg_namespace n ON n.oid = t.typnamespace
+        WHERE n.nspname = 'hectic' AND t.typname = 'migration_name'
+    ) THEN
+        CREATE DOMAIN "hectic"."migration_name" AS TEXT
+            CHECK (VALUE ~ '^[0-9]{14}-.*');
+    END IF;
+
+    IF NOT EXISTS (
+        SELECT 1
+        FROM pg_type t JOIN pg_namespace n ON n.oid = t.typnamespace
+        WHERE n.nspname = 'hectic' AND t.typname = 'sha256'
+    ) THEN
+        CREATE DOMAIN "hectic"."sha256" AS CHAR(64)
+            CHECK (VALUE ~ '^[0-9a-f]{64}$');
+    END IF;
+END
+$bootstrap$;
+
+CREATE OR REPLACE FUNCTION "hectic"."sha256_lower"() RETURNS trigger
+LANGUAGE plpgsql AS $fn$
+BEGIN
+    NEW."hash" := lower(NEW."hash");
+    RETURN NEW;
+END
+$fn$;
+
+CREATE TABLE IF NOT EXISTS "hectic"."migration" (
+    "id"          SERIAL                  PRIMARY KEY,
+    "name"        "hectic"."migration_name" UNIQUE NOT NULL,
+    "hash"        "hectic"."sha256"         UNIQUE NOT NULL,
+    "applied_at"  TIMESTAMPTZ             NOT NULL DEFAULT NOW()
+);
+
+DO $trg$
+BEGIN
+    IF NOT EXISTS (
+        SELECT 1 FROM pg_trigger
+        WHERE tgname = 'hectic_t_sha256_lower'
+          AND tgrelid = '"hectic"."migration"'::regclass
+          AND NOT tgisinternal
+    ) THEN
+        CREATE TRIGGER "hectic_t_sha256_lower"
+            BEFORE INSERT OR UPDATE ON "hectic"."migration"
+            FOR EACH ROW EXECUTE FUNCTION "hectic"."sha256_lower"();
+    END IF;
+END
+$trg$;

lib/hook/sql/hectic-secret.sql

@@ -0,0 +1,51 @@
+CREATE TABLE IF NOT EXISTS "hectic"."secret" (
+    "id"     SERIAL PRIMARY KEY,
+    "key"    TEXT   UNIQUE NOT NULL,
+    "value"  TEXT          NOT NULL
+);
+
+CREATE OR REPLACE FUNCTION "hectic"."load_secrets_from_env"(env_content TEXT)
+RETURNS void
+LANGUAGE plpgsql AS $fn$
+DECLARE
+    line TEXT;
+    k    TEXT;
+    v    TEXT;
+BEGIN
+    TRUNCATE TABLE "hectic"."secret";
+
+    FOR line IN
+        SELECT regexp_split_to_table(env_content, E'\n')
+    LOOP
+        line := btrim(line);
+
+        IF line = '' OR line LIKE '#%' THEN
+            CONTINUE;
+        END IF;
+
+        k := split_part(line, '=', 1);
+        v := substring(line FROM position('=' IN line) + 1);
+
+        k := btrim(k);
+        v := btrim(v);
+
+        IF v ~ '^".*"$' OR v ~ '^''.*''$' THEN
+            v := substring(v FROM 2 FOR char_length(v) - 2);
+        END IF;
+
+        INSERT INTO "hectic"."secret" ("key", "value") VALUES (k, v);
+    END LOOP;
+END
+$fn$;
+
+CREATE OR REPLACE FUNCTION "hectic"."get_secret"(k TEXT)
+RETURNS TEXT
+LANGUAGE plpgsql AS $fn$
+BEGIN
+    RETURN (
+        SELECT "value"
+        FROM "hectic"."secret"
+        WHERE "key" = k
+    );
+END
+$fn$;

lib/hook/sql/hectic-version.sql

@@ -0,0 +1,26 @@
+CREATE SCHEMA IF NOT EXISTS "hectic";
+
+CREATE TABLE IF NOT EXISTS "hectic"."version" (
+    "name"          TEXT                 PRIMARY KEY,
+    "version"       TEXT                 NOT NULL,
+    "installed_at"  TIMESTAMPTZ          NOT NULL DEFAULT NOW()
+);
+
+DO $check$
+DECLARE
+    existing TEXT;
+BEGIN
+    SELECT "version" INTO existing
+    FROM "hectic"."version"
+    WHERE "name" = 'hectic';
+
+    IF existing IS NULL THEN
+        INSERT INTO "hectic"."version" ("name", "version")
+        VALUES ('hectic', '@HECTIC_VERSION@');
+    ELSIF existing <> '@HECTIC_VERSION@' THEN
+        RAISE EXCEPTION
+            'hectic schema version mismatch: database has %, code expects %',
+            existing, '@HECTIC_VERSION@';
+    END IF;
+END
+$check$;

lib/parse-env.nix

@@ -0,0 +1,13 @@
+ file: let
+    envText = builtins.readFile file;
+    envLines = builtins.split "\n" envText;
+    lines = builtins.filter (line: (builtins.match "^.*=.*" line) != null) envLines;
+    #attributes = builtins.listToAttrs (builtins.map (line: let
+    #  parts = builtins.split "=" line;
+    #  key = builtins.substring 0 (builtins.stringLength parts[0] - 3) parts[0]; # Remove "var" prefix
+    #  value = parts[1];
+    #in {
+    #  name = key;
+    #  value = value;
+    #}) lines);
+  in { inherit envLines lines; }

lib/shell/check-tool.sh

@@ -0,0 +1,6 @@
+check_tool() {
+  if ! command -v "$1" >/dev/null; then
+    echo "Required tool \`$2\` are not installed or binary \`$1\` not found." >&2
+    exit 1
+  fi
+}

lib/shell/load-sops.sh

@@ -0,0 +1,204 @@
+load_sops_shell_quote() {
+  printf "'"
+  printf '%s' "$1" | sed "s/'/'\"'\"'/g"
+  printf "'"
+}
+
+load_sops_normalize_key() {
+  load_sops_normalized_key=$(printf '%s' "$1" | tr '.-' '__' | tr '[:lower:]' '[:upper:]')
+
+  case "$load_sops_normalized_key" in
+    ''|[!A-Z_]*|*[!A-Z0-9_]*)
+      printf 'load-sops: invalid environment name after key normalization\n' >&2
+      return 1
+      ;;
+  esac
+
+  printf '%s\n' "$load_sops_normalized_key"
+}
+
+load_sops_env_from_sops_file() {
+  load_sops_file=$1
+  load_sops_extract=${2-}
+
+  if ! command -v sops >/dev/null 2>&1; then
+    printf 'load-sops: required tool `sops` not found\n' >&2
+    return 1
+  fi
+
+  if ! command -v yq >/dev/null 2>&1; then
+    printf 'load-sops: required tool `yq` not found\n' >&2
+    return 1
+  fi
+
+  load_sops_decrypted=''
+  load_sops_attempt=0
+  load_sops_max_retries=${LOAD_SOPS_MAX_RETRIES:-1}
+  load_sops_prompt=${LOAD_SOPS_PROMPT:-0}
+
+  while :; do
+    if [ -n "$load_sops_extract" ]; then
+      if load_sops_decrypted=$(sops -d --extract "$load_sops_extract" "$load_sops_file" 2>/dev/null); then
+        load_sops_status=0
+      else
+        load_sops_status=$?
+      fi
+    else
+      if load_sops_decrypted=$(sops -d "$load_sops_file" 2>/dev/null); then
+        load_sops_status=0
+      else
+        load_sops_status=$?
+      fi
+    fi
+
+    if [ "$load_sops_status" -eq 0 ]; then
+      break
+    fi
+
+    if [ "$load_sops_prompt" != 1 ]; then
+      printf 'load-sops: failed to decrypt file\n' >&2
+      return 1
+    fi
+
+    if ! [ -t 0 ] || ! [ -r /dev/tty ]; then
+      printf 'load-sops: decrypt failed and prompt requested, but no TTY is available\n' >&2
+      return 1
+    fi
+
+    load_sops_attempt=$((load_sops_attempt + 1))
+    if [ "$load_sops_max_retries" != 0 ] && [ "$load_sops_attempt" -gt "$load_sops_max_retries" ]; then
+      printf 'load-sops: decrypt failed after configured retries\n' >&2
+      return 1
+    fi
+
+    load_sops_use_script=${LOAD_SOPS_USE_SCRIPT:-auto}
+    load_sops_quoted_file=$(load_sops_shell_quote "$load_sops_file") || return 1
+    load_sops_quoted_tty=$(load_sops_shell_quote "$(tty)") || return 1
+    case "$load_sops_use_script" in
+      auto)
+        if command -v script >/dev/null 2>&1 && [ -t 0 ]; then
+          if script -qefc "env GPG_TTY=$load_sops_quoted_tty sops --output /dev/null -d $load_sops_quoted_file" /dev/null >/dev/null 2>&1; then
+            load_sops_script_status=0
+          else
+            load_sops_script_status=$?
+          fi
+          if [ "$load_sops_script_status" -eq 0 ]; then
+            continue
+          fi
+        fi
+        ;;
+      1)
+        if ! command -v script >/dev/null 2>&1; then
+          printf 'load-sops: required tool `script` not found\n' >&2
+          return 1
+        fi
+        if script -qefc "env GPG_TTY=$load_sops_quoted_tty sops --output /dev/null -d $load_sops_quoted_file" /dev/null >/dev/null 2>&1; then
+          load_sops_script_status=0
+        else
+          load_sops_script_status=$?
+        fi
+        if [ "$load_sops_script_status" -eq 0 ]; then
+          continue
+        fi
+        ;;
+      0)
+        ;;
+      *)
+        printf 'load-sops: LOAD_SOPS_USE_SCRIPT must be auto, 0, or 1\n' >&2
+        return 1
+        ;;
+    esac
+
+    printf 'load-sops: enter SOPS_AGE_KEY_CMD: ' >/dev/tty
+    if ! IFS= read -r SOPS_AGE_KEY_CMD </dev/tty; then
+      printf 'load-sops: failed to read prompt input\n' >&2
+      return 1
+    fi
+    export SOPS_AGE_KEY_CMD
+  done
+
+  load_sops_env_from_yaml_text "$load_sops_decrypted"
+}
+
+load_sops_env_from_yaml_file() {
+  load_sops_file=$1
+
+  if ! command -v yq >/dev/null 2>&1; then
+    printf 'load-sops: required tool `yq` not found\n' >&2
+    return 1
+  fi
+
+  load_sops_env_from_yaml_text "$(cat "$load_sops_file")"
+}
+
+load_sops_env_from_yaml_text() {
+  load_sops_yaml=$1
+  load_sops_seen=''
+
+  if load_sops_keys=$(printf '%s' "$load_sops_yaml" | yq -r 'keys | .[]' 2>/dev/null); then
+    load_sops_keys_status=0
+  else
+    load_sops_keys_status=$?
+  fi
+
+  if [ "$load_sops_keys_status" -ne 0 ]; then
+    printf 'load-sops: failed to inspect YAML top-level keys\n' >&2
+    return 1
+  fi
+
+  while IFS= read -r load_sops_key; do
+    [ -n "$load_sops_key" ] || continue
+
+    load_sops_name=$(load_sops_normalize_key "$load_sops_key") || return 1
+
+    case "
+$load_sops_seen
+" in
+      *"
+$load_sops_name
+"*)
+        if [ "${LOAD_SOPS_ALLOW_COLLISIONS:-0}" != 1 ]; then
+          printf 'load-sops: normalized environment name collision\n' >&2
+          return 1
+        fi
+        ;;
+    esac
+    load_sops_seen=${load_sops_seen}${load_sops_seen:+"
+"}$load_sops_name
+
+    load_sops_kind=$(printf '%s' "$load_sops_yaml" | yq -r '."'"$load_sops_key"'" | kind' 2>/dev/null) || {
+      printf 'load-sops: failed to inspect YAML value kind\n' >&2
+      return 1
+    }
+    load_sops_tag=$(printf '%s' "$load_sops_yaml" | yq -r '."'"$load_sops_key"'" | tag' 2>/dev/null) || {
+      printf 'load-sops: failed to inspect YAML value tag\n' >&2
+      return 1
+    }
+
+    if [ "$load_sops_kind" != scalar ]; then
+      printf 'load-sops: top-level YAML values must be scalars\n' >&2
+      return 1
+    fi
+
+    if [ "$load_sops_tag" = '!!null' ]; then
+      printf 'load-sops: top-level YAML null values are not supported\n' >&2
+      return 1
+    fi
+
+    if [ "${LOAD_SOPS_OVERWRITE:-1}" = 0 ]; then
+      eval 'load_sops_already_set=${'"$load_sops_name"'+x}'
+      if [ -n "$load_sops_already_set" ]; then
+        continue
+      fi
+    fi
+
+    load_sops_value=$(printf '%s' "$load_sops_yaml" | yq -r '."'"$load_sops_key"'"' 2>/dev/null) || {
+      printf 'load-sops: failed to read YAML scalar value\n' >&2
+      return 1
+    }
+    load_sops_quoted=$(load_sops_shell_quote "$load_sops_value") || return 1
+    eval "export $load_sops_name=$load_sops_quoted"
+  done <<EOF
+$load_sops_keys
+EOF
+}

lib/shell/local-dir.sh

@@ -0,0 +1,14 @@
+printf '\033[0;34mDetecting local directories...\033[0m\n'
+if git_root=$($BIN_GIT rev-parse --show-toplevel 2>/dev/null); then
+  LOCAL_DIR="$git_root"
+  printf '\033[0;32mFound git root: \033[1;37m%s\033[0m\n' "$LOCAL_DIR"
+else
+  LOCAL_DIR="$(pwd)"
+  printf '\033[1;33mNot in git repo, using current dir: \033[1;37m%s\033[0m\n' "$LOCAL_DIR"
+  printf 'Are you realy want continue? (y/n):\n'
+  read -r CONTINUE
+  if [ "$CONTINUE" != "y" ]; then
+    printf '\033[0;31mAborting...\033[0m\n'
+    exit 0
+  fi
+fi

lib/shell/logs.sh

@@ -0,0 +1,23 @@
+RED='\033[0;31m'
+GREEN='\033[0;32m'
+YELLOW='\033[1;33m'
+BLUE='\033[0;34m'
+PURPLE='\033[0;35m'
+MAGENTA="$PURPLE"
+CYAN='\033[0;36m'
+WHITE='\033[1;37m'
+NC='\033[0m' # No Color
+
+LOG_PATH="/var/log/hectic/activation.log"
+
+if ! mkdir -p "$(dirname "$LOG_PATH")" 2>/dev/null; then
+  LOG_PATH="/dev/null"
+fi
+
+log_info()    { text=$1; shift; printf "%b ${text}%b\n" "$BLUE"   "$@" "$NC" | tee -a "$LOG_PATH" >&2; }
+log_success() { text=$1; shift; printf "%b ${text}%b\n" "$GREEN"  "$@" "$NC" | tee -a "$LOG_PATH" >&2; }
+log_warning() { text=$1; shift; printf "%b ${text}%b\n" "$YELLOW" "$@" "$NC" | tee -a "$LOG_PATH" >&2; }
+log_error()   { text=$1; shift; printf "%b ${text}%b\n" "$RED"    "$@" "$NC" | tee -a "$LOG_PATH" >&2; }
+log_step()    { text=$1; shift; printf "%b ${text}%b\n" "$PURPLE" "$@" "$NC" | tee -a "$LOG_PATH" >&2; }
+
+log_header() { printf "\n%b=== %s ===%b\n" "$WHITE" "$@" "$NC" | tee -a "$LOG_PATH" >&2; }

nixos/module/default.nix

@@ -0,0 +1,21 @@
+{
+  flake,
+  self,
+  inputs,
+}:
+with builtins;
+with inputs.nixpkgs.lib;
+with self.lib;
+let
+  # Combine hectic modules into one
+  hectic.imports = attrValues (
+    readModulesRecursive' (flake + "/nixos/module/hectic") { inherit flake self inputs; }
+  );
+  # Read generic modules separately
+  generic = readModulesRecursive'
+    (flake + "/nixos/module/generic")
+    { inherit flake self inputs; };
+in generic // {
+  inherit hectic;
+  default = hectic;
+}

nixos/module/generic/matrix-cluster-users.nix

@@ -0,0 +1,48 @@
+{
+  inputs,
+  flake,
+  self,
+}: {
+  lib,
+  config,
+  ...
+}: let
+  userNames = [
+    "yukkop"
+    "liquiz"
+    "vismajor"
+    "lvgkcfjl"
+    "MrAlex0O"
+    "Антоша"
+    "snuff"
+  ];
+
+  adminNames = [ "yukkop" ];
+  matrixClusterSopsFile = flake + "/sus/matrix-cluster.yaml";
+
+  cfg = config.hectic.generic.matrix-cluster;
+in {
+  config = lib.mkIf cfg.enable {
+    hectic.generic.matrix-cluster.users = builtins.listToAttrs (
+      map (name: {
+        inherit name;
+        value = {
+          passwordFile = config.sops.secrets."matrix/users/${name}/password".path;
+        } // lib.optionalAttrs (builtins.elem name adminNames) {
+          admin = true;
+        };
+      }) userNames
+    );
+
+    sops.secrets = builtins.listToAttrs (
+      map (name: {
+        name = "matrix/users/${name}/password";
+        value = {
+          key      = "matrix/users/${name}/password";
+          owner    = "matrix-synapse";
+          sopsFile = matrixClusterSopsFile;
+        };
+      }) userNames
+    );
+  };
+}

nixos/module/generic/matrix-cluster.nix

@@ -0,0 +1,599 @@
+{
+  inputs,
+  flake,
+  self,
+}: {
+  pkgs,
+  lib,
+  config,
+  ...
+}: let
+  cfg = config.hectic.generic.matrix-cluster;
+  s3Cfg = cfg.objectStorage.s3;
+
+  s3Plugin = pkgs.matrix-synapse-plugins.matrix-synapse-s3-storage-provider;
+  s3ConfigDir = "/run/matrix-synapse";
+  s3ConfigFile = "${s3ConfigDir}/s3-media-storage.yaml";
+
+  pgDataDir = "/var/lib/postgresql/17";
+
+  matrixUsers = builtins.attrNames cfg.users;
+
+  mkUserRegistration = name: let
+    user = cfg.users.${name};
+    adminFlag = if user.admin then "--admin" else "--no-admin";
+  in ''
+    if [ ! -r "${user.passwordFile}" ]; then
+      printf 'Missing Matrix password file for %s: %s\n' '${name}' '${user.passwordFile}' >&2
+      exit 1
+    fi
+
+    ${pkgs.matrix-synapse}/bin/register_new_matrix_user \
+      -u '${name}' \
+      -p "$(tr -d '\n' < "${user.passwordFile}")" \
+      -k "$REGISTRATION_SHARED_SECRET" \
+      ${adminFlag} \
+      http://127.0.0.1:8008 || true
+  '';
+
+  synapseEnabled =
+    if cfg.overrideEnableSynapse != null
+    then cfg.overrideEnableSynapse
+    else cfg.role == "primary";
+
+  mkS3Config = ''
+    if [ ! -r "${s3Cfg.credentialsFile}" ]; then
+      printf 'Missing Matrix object storage credentials file: %s\n' '${s3Cfg.credentialsFile}' >&2
+      exit 1
+    fi
+
+    . "${s3Cfg.credentialsFile}"
+
+    if [ -z "$ACCESS_KEY_ID" ] || [ -z "$SECRET_ACCESS_KEY" ]; then
+      printf 'ACCESS_KEY_ID or SECRET_ACCESS_KEY missing in %s\n' '${s3Cfg.credentialsFile}' >&2
+      exit 1
+    fi
+
+    mkdir -p "${s3ConfigDir}"
+
+    cat > "${s3ConfigFile}" <<EOF
+    media_storage_providers:
+      - module: s3_storage_provider.S3StorageProviderBackend
+        store_local: ${lib.boolToString s3Cfg.storeLocal}
+        store_remote: ${lib.boolToString s3Cfg.storeRemote}
+        store_synchronous: ${lib.boolToString s3Cfg.storeSynchronous}
+        config:
+          bucket: ${s3Cfg.bucket}
+          endpoint_url: ${s3Cfg.endpointUrl}
+          region_name: ${s3Cfg.regionName}
+          prefix: "${s3Cfg.prefix}"
+          storage_class: "${s3Cfg.storageClass}"
+          threadpool_size: ${toString s3Cfg.threadpoolSize}
+          access_key_id: $ACCESS_KEY_ID
+          secret_access_key: $SECRET_ACCESS_KEY
+    EOF
+
+    chown matrix-synapse:matrix-synapse "${s3ConfigFile}"
+    chmod 0400 "${s3ConfigFile}"
+  '';
+in {
+  options.hectic.generic.matrix-cluster = {
+    enable = lib.mkEnableOption "Matrix Synapse active/passive cluster node";
+
+    role = lib.mkOption {
+      type = lib.types.enum [ "primary" "standby" ];
+      description = ''
+        Cluster role of this node. The primary runs Synapse and accepts WAL
+        streaming connections; the standby runs a hot-standby Postgres replica
+        only and keeps Synapse disabled until failover.
+      '';
+    };
+
+    matrixDomain = lib.mkOption {
+      type = lib.types.str;
+      description = "Matrix server_name (also nginx vhost / ACME cert name).";
+    };
+
+    signingKeyFile = lib.mkOption {
+      type = lib.types.path;
+      description = ''
+        Path to the Synapse homeserver signing key. Mounted into place at
+        /var/lib/matrix-synapse/homeserver.signing.key on activation.
+      '';
+    };
+
+    secretsFile = lib.mkOption {
+      type = lib.types.nullOr lib.types.path;
+      default = null;
+      description = ''
+        Extra Synapse YAML config (registration_shared_secret, macaroon_secret_key,
+        form_secret). Loaded via matrix-synapse extraConfigFiles. Required when
+        Synapse is enabled on this node (primary, or standby after failover).
+      '';
+    };
+
+    turnSecretFile = lib.mkOption {
+      type = lib.types.nullOr lib.types.path;
+      default = null;
+      description = ''
+        Shared secret file used by coturn for Matrix voice/video calls.
+        When set together with `publicIp`, the active Synapse node also enables
+        coturn and publishes TURN URIs to clients.
+      '';
+    };
+
+    publicIp = lib.mkOption {
+      type = lib.types.nullOr lib.types.str;
+      default = null;
+      description = ''
+        Public IP address advertised to coturn for listening and relaying.
+      '';
+    };
+
+    maxUploadSize = lib.mkOption {
+      type = lib.types.str;
+      default = "2G";
+    };
+
+    enableRegistration = lib.mkOption {
+      type = lib.types.bool;
+      default = false;
+    };
+
+    users = lib.mkOption {
+      type = lib.types.attrsOf (lib.types.submodule {
+        options = {
+          passwordFile = lib.mkOption { type = lib.types.str; };
+          admin = lib.mkOption { type = lib.types.bool; default = false; };
+        };
+      });
+      default = {};
+      description = "Declarative Matrix users provisioned via register_new_matrix_user.";
+    };
+
+    overrideEnableSynapse = lib.mkOption {
+      type = lib.types.nullOr lib.types.bool;
+      default = null;
+      description = ''
+        When non-null, forces Synapse on/off regardless of role. Used during
+        failover: set to true on the standby once it has been promoted, or
+        false on the primary to drain it.
+      '';
+    };
+
+    objectStorage.s3 = {
+      bucket = lib.mkOption { type = lib.types.str; };
+      regionName = lib.mkOption { type = lib.types.str; };
+      endpointUrl = lib.mkOption { type = lib.types.str; };
+      credentialsFile = lib.mkOption {
+        type = lib.types.path;
+        description = ''
+          env-style file with ACCESS_KEY_ID= and SECRET_ACCESS_KEY=. MUST be
+          the SAME credentials/bucket on both primary and standby.
+        '';
+      };
+      mediaStorePath = lib.mkOption {
+        type = lib.types.str;
+        default = "/var/lib/matrix-synapse/media_store";
+      };
+      prefix = lib.mkOption { type = lib.types.str; default = ""; };
+      storageClass = lib.mkOption { type = lib.types.str; default = "STANDARD"; };
+      threadpoolSize = lib.mkOption { type = lib.types.int; default = 40; };
+      storeLocal = lib.mkOption { type = lib.types.bool; default = true; };
+      storeRemote = lib.mkOption { type = lib.types.bool; default = true; };
+      storeSynchronous = lib.mkOption { type = lib.types.bool; default = true; };
+    };
+
+    replication = {
+      peerHost = lib.mkOption {
+        type = lib.types.str;
+        description = "Public IP/hostname of the other cluster node.";
+      };
+      peerPort = lib.mkOption {
+        type = lib.types.port;
+        default = 5432;
+      };
+      passwordFile = lib.mkOption {
+        type = lib.types.path;
+        description = ''
+          File containing either a raw replication password or a libpq passfile
+          line. Used as `passfile=` in primary_conninfo on the standby and to
+          set the password of the `replication` Postgres role on the primary.
+        '';
+      };
+      allowedSourceIPs = lib.mkOption {
+        type = lib.types.listOf lib.types.str;
+        default = [];
+        description = ''
+          CIDRs allowed to connect to Postgres for replication. Used on the
+          primary in pg_hba.conf hostssl entries and to gate the firewall.
+        '';
+      };
+      sslMode = lib.mkOption {
+        type = lib.types.str;
+        default = "require";
+      };
+    };
+
+    acme = {
+      enable = lib.mkEnableOption "Porkbun DNS-01 ACME for matrixDomain";
+      email = lib.mkOption {
+        type = lib.types.str;
+        default = "hectic.yukkop.it@gmail.com";
+        description = "ACME registration email (passed to security.acme.defaults.email).";
+      };
+      porkbunApiKeyFile = lib.mkOption {
+        type = lib.types.path;
+        description = "File containing PORKBUN_API_KEY value.";
+      };
+      porkbunSecretApiKeyFile = lib.mkOption {
+        type = lib.types.path;
+        description = "File containing PORKBUN_SECRET_API_KEY value.";
+      };
+    };
+
+    jitsi.preferredDomain = lib.mkOption {
+      type = lib.types.nullOr lib.types.str;
+      default = null;
+      description = ''
+        Optional self-hosted Jitsi Meet domain to advertise to Matrix/Element
+        clients alongside the cluster-managed homeserver.
+      '';
+    };
+  };
+
+  config = lib.mkIf cfg.enable (lib.mkMerge [
+
+    {
+      # signing key mount: copy into matrix-synapse data dir with correct perms
+      # regardless of whether Synapse is currently enabled on this node, so a
+      # failover flip does not need a separate provisioning step.
+      systemd.tmpfiles.rules = [
+        "d /var/lib/matrix-synapse 0750 matrix-synapse matrix-synapse -"
+        "Z ${s3Cfg.mediaStorePath} 0700 matrix-synapse matrix-synapse -"
+      ];
+
+      systemd.services.matrix-cluster-signing-key = {
+        description = "Install Matrix Synapse signing key from secrets";
+        wantedBy = [ "multi-user.target" ];
+        before = lib.optional synapseEnabled "matrix-synapse.service";
+        requiredBy = lib.optional synapseEnabled "matrix-synapse.service";
+        serviceConfig = {
+          Type = "oneshot";
+          RemainAfterExit = true;
+        };
+        script = ''
+          set -eu
+          install -d -o matrix-synapse -g matrix-synapse -m 0750 /var/lib/matrix-synapse
+          install -o matrix-synapse -g matrix-synapse -m 0400 \
+            "${cfg.signingKeyFile}" \
+            /var/lib/matrix-synapse/homeserver.signing.key
+        '';
+      };
+
+      users.users.matrix-synapse = {
+        isSystemUser = true;
+        group = "matrix-synapse";
+      };
+      users.groups.matrix-synapse = {};
+    }
+
+    (lib.mkIf synapseEnabled {
+      assertions = [
+        {
+          assertion = cfg.secretsFile != null;
+          message = "hectic.generic.matrix-cluster.secretsFile must be set when Synapse runs on this node.";
+        }
+        {
+          assertion = (cfg.turnSecretFile == null) == (cfg.publicIp == null);
+          message = "hectic.generic.matrix-cluster.turnSecretFile and publicIp must be set together.";
+        }
+      ];
+
+      services.coturn = lib.mkIf (cfg.turnSecretFile != null) rec {
+        enable = true;
+        realm = cfg.matrixDomain;
+        use-auth-secret = true;
+        static-auth-secret-file = cfg.turnSecretFile;
+        cert = "${config.security.acme.certs.${realm}.directory}/full.pem";
+        pkey = "${config.security.acme.certs.${realm}.directory}/key.pem";
+        listening-ips = [ cfg.publicIp ];
+        relay-ips = [ cfg.publicIp ];
+        listening-port = 3478;
+        tls-listening-port = 5349;
+        no-cli = true;
+
+        extraConfig = ''
+          verbose
+        '';
+      };
+
+      services.matrix-synapse = {
+        enable = true;
+        plugins = [ s3Plugin ];
+        extraConfigFiles = [ cfg.secretsFile s3ConfigFile ];
+
+        settings = {
+          server_name = cfg.matrixDomain;
+          public_baseurl = "https://${cfg.matrixDomain}";
+          max_upload_size = cfg.maxUploadSize;
+          media_store_path = s3Cfg.mediaStorePath;
+          signing_key_path = "/var/lib/matrix-synapse/homeserver.signing.key";
+
+          # Tolerate bursty Element/iPhone presence syncs without disabling limits.
+          rc_presence.per_user = {
+            per_second = 0.5;
+            burst_count = 5;
+          };
+
+          experimental_features = {
+            msc3266_enabled = true;
+            msc4140_enabled = true;
+            msc4143_enabled = true;
+            msc4222_enabled = true;
+          };
+
+          matrix_rtc = {
+            transports = [
+              {
+                type = "livekit";
+                livekit_service_url = "https://${cfg.matrixDomain}/livekit/jwt";
+              }
+            ];
+          };
+
+          listeners = [
+            {
+              port = 8008;
+              bind_addresses = [ "0.0.0.0" ];
+              type = "http";
+              tls = false;
+              resources = [
+                {
+                  names = [ "client" "federation" "openid" ];
+                  compress = false;
+                }
+              ];
+            }
+          ];
+
+          enable_registration = cfg.enableRegistration;
+          enable_registration_without_verification = cfg.enableRegistration;
+        } // lib.optionalAttrs (cfg.turnSecretFile != null) {
+          turn_uris = [
+            "turn:${cfg.matrixDomain}:3478?transport=udp"
+            "turn:${cfg.matrixDomain}:3478?transport=tcp"
+            "turns:${cfg.matrixDomain}:5349?transport=udp"
+            "turns:${cfg.matrixDomain}:5349?transport=tcp"
+          ];
+          turn_user_lifetime = 86400000;
+          turn_allow_guests = true;
+        };
+      };
+
+      environment.systemPackages = [ pkgs.matrix-synapse ];
+
+      systemd.services.matrix-synapse-s3-config = {
+        description = "Generate Synapse S3 media storage config";
+        before = [ "matrix-synapse.service" ];
+        requiredBy = [ "matrix-synapse.service" ];
+        serviceConfig.Type = "oneshot";
+        script = mkS3Config;
+      };
+
+      services.nginx = {
+        enable = true;
+        virtualHosts.${cfg.matrixDomain} = {
+          forceSSL = true;
+          enableACME = true;
+          locations."/" = {
+            proxyPass = "http://127.0.0.1:8008";
+            extraConfig = ''
+              client_max_body_size ${cfg.maxUploadSize};
+            '';
+          };
+          locations."=/.well-known/matrix/server" = {
+            extraConfig = ''
+              default_type application/json;
+              add_header Access-Control-Allow-Origin *;
+              add_header Access-Control-Allow-Methods "GET, POST, PUT, DELETE, OPTIONS";
+              add_header Access-Control-Allow-Headers "X-Requested-With, Content-Type, Authorization";
+            '';
+            return = "200 '{\"m.server\": \"${cfg.matrixDomain}:443\"}'";
+          };
+        };
+      };
+
+      networking.firewall = lib.mkIf (cfg.turnSecretFile != null) {
+        allowedUDPPorts = [ 3478 5349 ];
+        allowedTCPPorts = [ 3478 5349 ];
+        allowedTCPPortRanges = [
+          {
+            from = 49152;
+            to = 65535;
+          }
+        ];
+        allowedUDPPortRanges = [
+          {
+            from = 49152;
+            to = 65535;
+          }
+        ];
+      };
+
+      systemd.services.matrix-synapse-users = lib.mkIf (matrixUsers != []) {
+        description = "Provision Matrix Synapse users";
+        wantedBy = [ "multi-user.target" ];
+        after = [ "matrix-synapse.service" ];
+        requires = [ "matrix-synapse.service" ];
+        path = with pkgs; [ curl coreutils gawk ];
+        serviceConfig = {
+          Type = "oneshot";
+          User = "matrix-synapse";
+        };
+        script = ''
+          until curl -sf http://127.0.0.1:8008/_matrix/client/versions >/dev/null; do
+            sleep 2
+          done
+
+          REGISTRATION_SHARED_SECRET="$(awk -F': *' '$1 == "registration_shared_secret" { print $2; exit }' "${cfg.secretsFile}")"
+
+          if [ -z "$REGISTRATION_SHARED_SECRET" ]; then
+            printf 'registration_shared_secret not found in %s\n' '${cfg.secretsFile}' >&2
+            exit 1
+          fi
+
+${lib.concatStringsSep "\n" (map mkUserRegistration matrixUsers)}
+        '';
+      };
+    })
+
+    {
+      services.postgresql = {
+        enable = true;
+        package = pkgs.postgresql_17;
+        enableTCPIP = true;
+
+        initdbArgs = [ "--locale=C" "--encoding=UTF8" ];
+
+        settings = {
+          wal_level = "replica";
+          max_wal_senders = 4;
+          hot_standby = "on";
+        };
+      };
+    }
+
+    (lib.mkIf (cfg.role == "primary") {
+      services.postgresql = {
+        authentication = lib.concatStringsSep "\n" ([
+          "local all         all                          trust"
+          "host  sameuser    all          127.0.0.1/32    scram-sha-256"
+          "host  sameuser    all          ::1/128         scram-sha-256"
+          "host  all         all          ::1/128         scram-sha-256"
+          "host  all         all          0.0.0.0/0       scram-sha-256"
+          "host  replication postgres     127.0.0.1/32    scram-sha-256"
+          "host  replication postgres     ::1/128         scram-sha-256"
+        ] ++ map (cidr:
+          "hostssl replication replication ${cidr} scram-sha-256"
+        ) cfg.replication.allowedSourceIPs);
+
+        ensureUsers = [
+          {
+            name = "replication";
+            ensureClauses = {
+              login = true;
+              replication = true;
+            };
+          }
+        ];
+      };
+
+      # Apply replication password from SOPS-mounted file after postgres start.
+      systemd.services.matrix-cluster-replication-password = {
+        description = "Set Postgres replication role password from SOPS";
+        wantedBy = [ "multi-user.target" ];
+        after = [ "postgresql.service" ];
+        requires = [ "postgresql.service" ];
+        serviceConfig = {
+          Type = "oneshot";
+          User = "postgres";
+          RemainAfterExit = true;
+        };
+        script = ''
+          set -eu
+          PW="$(tr -d '\n' < "${cfg.replication.passwordFile}")"
+          ${config.services.postgresql.package}/bin/psql -v ON_ERROR_STOP=1 -c \
+            "ALTER ROLE replication WITH LOGIN REPLICATION PASSWORD '$PW';"
+        '';
+      };
+    })
+
+    (lib.mkIf (cfg.role == "standby") {
+      systemd.targets.postgresql.requires = lib.mkForce [
+        "postgresql.service"
+      ];
+
+      # Hot-standby bootstrap: standby.signal + primary_conninfo with passfile.
+      # pg_basebackup must be run manually (see runbook) before this activates
+      # for the first time.
+      systemd.services.matrix-cluster-standby-bootstrap = {
+        description = "Configure Matrix Postgres hot standby";
+        wantedBy = [ "postgresql.service" ];
+        before = [ "postgresql.service" ];
+        serviceConfig = {
+          Type = "oneshot";
+          RemainAfterExit = true;
+        };
+        script = ''
+          set -eu
+          if [ ! -d "${pgDataDir}" ]; then
+            echo "Postgres data dir ${pgDataDir} missing; run pg_basebackup first (see MATRIX-FAILOVER-RUNBOOK.md)" >&2
+            exit 0
+          fi
+
+          # Materialize a libpq passfile from the raw password secret.
+          PASSFILE=/var/lib/postgresql/.matrix-cluster-replication.passfile
+          PW="$(tr -d '\n' < "${cfg.replication.passwordFile}")"
+          umask 077
+          printf '%s:%d:replication:replication:%s\n' \
+            '${cfg.replication.peerHost}' \
+            ${toString cfg.replication.peerPort} \
+            "$PW" > "$PASSFILE"
+          chown postgres:postgres "$PASSFILE"
+          chmod 0600 "$PASSFILE"
+
+          touch "${pgDataDir}/standby.signal"
+          chown postgres:postgres "${pgDataDir}/standby.signal"
+
+          CONF="${pgDataDir}/postgresql.auto.conf"
+          touch "$CONF"
+          chown postgres:postgres "$CONF"
+          # Strip any prior primary_conninfo line, then append fresh one.
+          ${pkgs.gnused}/bin/sed -i '/^primary_conninfo/d' "$CONF"
+          printf "primary_conninfo = 'host=%s port=%d user=replication passfile=%s sslmode=%s'\n" \
+            '${cfg.replication.peerHost}' \
+            ${toString cfg.replication.peerPort} \
+            "$PASSFILE" \
+            '${cfg.replication.sslMode}' >> "$CONF"
+        '';
+      };
+    })
+
+    (lib.mkIf cfg.acme.enable {
+      security.acme = {
+        acceptTerms = true;
+        defaults.email = lib.mkDefault cfg.acme.email;
+        certs.${cfg.matrixDomain} = {
+          dnsProvider = "porkbun";
+          webroot = lib.mkForce null;
+          environmentFile = "/run/matrix-cluster/porkbun.env";
+        };
+      };
+
+      systemd.services.matrix-cluster-acme-env = {
+        description = "Assemble Porkbun ACME environment file";
+        wantedBy = [ "multi-user.target" ];
+        before = [ "acme-${cfg.matrixDomain}.service" ];
+        requiredBy = [ "acme-${cfg.matrixDomain}.service" ];
+        serviceConfig = {
+          Type = "oneshot";
+          RemainAfterExit = true;
+        };
+        script = ''
+          set -eu
+          install -d -m 0755 /run/matrix-cluster
+          API="$(tr -d '\n' < "${cfg.acme.porkbunApiKeyFile}")"
+          SEC="$(tr -d '\n' < "${cfg.acme.porkbunSecretApiKeyFile}")"
+          OUT=/run/matrix-cluster/porkbun.env
+          umask 077
+          {
+            printf 'PORKBUN_API_KEY=%s\n' "$API"
+            printf 'PORKBUN_SECRET_API_KEY=%s\n' "$SEC"
+          } > "$OUT"
+          chmod 0400 "$OUT"
+        '';
+      };
+    })
+  ]);
+}

nixos/module/generic/postgresql.nix

@@ -0,0 +1,97 @@
+{ 
+  inputs,
+  self,
+  flake
+}:
+{ 
+  pkgs,
+  config,
+  lib,
+  ...
+}: let
+  system = pkgs.stdenv.hostPlatform.system;
+  cfg = config.services.postgresql;
+  extensionFlags = {
+    pg_cron = false;
+    pgjwt = false;
+    pg_net = false;
+    pg_smtp_client = false;
+    http = false;
+    plsh = false;
+    hemar = false;
+  };
+in {
+  options = {
+    services.postgresql = {
+      lazzyExtensions = lib.mkOption {
+        type = lib.types.attrsOf lib.types.bool;
+        default = extensionFlags;
+      };
+      environment = lib.mkOption {
+        type    = lib.types.attrsOf lib.types.str;
+        default = {};
+      };
+      script = lib.mkOption {
+        type = with lib; types.nullOr types.path;
+        default = null;
+        example = lib.literalExpression ''
+          pkgs.writeText "init-sql-script" '''
+            alter user postgres with password 'myPassword';
+          ''';'';
+
+        description = ''
+          A file containing SQL statements to execute on stratup or any time you change it.
+        '';
+      };
+    };
+  };
+  config = lib.mkIf cfg.enable {
+    systemd.services.postgresql-script= lib.mkIf (cfg.script != null) {
+      description = "Some postgresql settings";
+      after = [ "postgresql.service" ];
+      wants = [ "postgresql.service" ];
+      serviceConfig = {
+        Type = "oneshot";
+        ExecStart = "${pkgs.dash}/bin/dash ${pkgs.writeText "sql-script" ''
+          #!/${pkgs.dash}/bin/dash 
+
+          set -e
+
+          alias psql='${cfg.package}/bin/psql -v ON_ERROR_STOP=1 -p "${builtins.toString cfg.port}" -U postgres -d postgres'
+
+          ${builtins.readFile cfg.script}
+        ''}";
+      };
+      path = [ cfg.package ];
+      wantedBy = [ "multi-user.target" ];
+    };
+    systemd.services.postgresql.environment = cfg.environment;
+    #services.postgresql = {
+    #  settings.shared_preload_libraries =
+    #    lib.concatStringsSep ", "
+    #      (lib.attrNames (
+  	#    lib.filterAttrs (n: v: v && 
+  	#         n != "http" 
+  	#      && n != "plsh" 
+  	#      && n != "pgjwt" 
+  	#      && n != "pg_smtp_client"
+  	#  ) cfg.lazzyExtensions));
+
+    #  extensions = let
+    #    packages = {
+    #      inherit (cfg.package.pkgs) pg_net pgjwt pg_cron http pg_smtp_client plsh;
+    #    };
+    #  in
+    #    lib.attrValues (
+    #      lib.filterAttrs (n: v: v != null)
+    #      (lib.mapAttrs' (
+    #          name: enabled:
+    #            if enabled
+    #            then lib.nameValuePair name (packages.${name} or (throw "Package ${name} not found in pkgs"))
+    #            else null
+    #        )
+    #        cfg.lazzyExtensions)
+    #    );
+    #};
+  };
+}

nixos/module/generic/shadowsocks-rust.nix

@@ -0,0 +1,169 @@
+# INFO(nrv): This is standalone shadowsocks module. Instance-specific is at ./shadowsocks.nix
+{
+  ...
+}:
+{
+  config,
+  lib,
+  pkgs,
+  ...
+}:
+
+with lib;
+
+let
+  cfg = config.services.shadowsocks-rust;
+
+  opts = {
+    server = cfg.localAddress;
+    server_port = cfg.port;
+    method = cfg.encryptionMethod;
+    mode = cfg.mode;
+    user = "nobody";
+    fast_open = cfg.fastOpen;
+  } // optionalAttrs (cfg.plugin != null) {
+    plugin = cfg.plugin;
+    plugin_opts = cfg.pluginOpts;
+  } // optionalAttrs (cfg.password != null) {
+    password = cfg.password;
+  } // cfg.extraConfig;
+
+  configFile = pkgs.writeText "shadowsocks.json" (builtins.toJSON opts);
+
+in
+
+{
+
+  ###### interface
+
+  options = {
+
+    services.shadowsocks-rust = {
+
+      enable = mkOption {
+        type = types.bool;
+        default = false;
+        description = lib.mdDoc ''
+          Whether to run shadowsocks-rust shadowsocks server.
+        '';
+      };
+
+      localAddress = mkOption {
+        type = types.str;
+        default =  "0.0.0.0";
+        description = lib.mdDoc ''
+          Local addresses to which the server binds.
+        '';
+      };
+
+      port = mkOption {
+        type = types.port;
+        default = 8388;
+        description = lib.mdDoc ''
+          Port which the server uses.
+        '';
+      };
+
+      password = mkOption {
+        type = types.nullOr types.str;
+        default = null;
+        description = lib.mdDoc ''
+          Password for connecting clients.
+        '';
+      };
+
+      passwordFile = mkOption {
+        type = types.nullOr types.path;
+        default = null;
+        description = lib.mdDoc ''
+          Password file with a password for connecting clients.
+        '';
+      };
+
+      mode = mkOption {
+        type = types.enum [ "tcp_only" "tcp_and_udp" "udp_only" ];
+        default = "tcp_and_udp";
+        description = lib.mdDoc ''
+          Relay protocols.
+        '';
+      };
+
+      fastOpen = mkOption {
+        type = types.bool;
+        default = true;
+        description = lib.mdDoc ''
+          use TCP fast-open
+        '';
+      };
+
+      encryptionMethod = mkOption {
+        type = types.str;
+        default = "chacha20-ietf-poly1305";
+        description = lib.mdDoc ''
+          Encryption method. See <https://github.com/shadowsocks/shadowsocks-org/wiki/AEAD-Ciphers>.
+        '';
+      };
+
+      plugin = mkOption {
+        type = types.nullOr types.str;
+        default = null;
+        example = literalExpression ''"''${pkgs.shadowsocks-v2ray-plugin}/bin/v2ray-plugin"'';
+        description = lib.mdDoc ''
+          SIP003 plugin for shadowsocks
+        '';
+      };
+
+      pluginOpts = mkOption {
+        type = types.str;
+        default = "";
+        example = "server;host=example.com";
+        description = lib.mdDoc ''
+          Options to pass to the plugin if one was specified
+        '';
+      };
+
+      extraConfig = mkOption {
+        type = types.attrs;
+        default = {};
+        example = {
+          nameserver = "8.8.8.8";
+        };
+        description = lib.mdDoc ''
+          Additional configuration for shadowsocks that is not covered by the
+          provided options. The provided attrset will be serialized to JSON and
+          has to contain valid shadowsocks options. Unfortunately most
+          additional options are undocumented but it's easy to find out what is
+          available by looking into the source code of
+          <https://github.com/shadowsocks/shadowsocks-rust/blob/master/src/jconf.c>
+        '';
+      };
+    };
+
+  };
+
+
+  ###### implementation
+
+  config = mkIf cfg.enable {
+    assertions = singleton
+      { assertion = cfg.password == null || cfg.passwordFile == null;
+        message = "Cannot use both password and passwordFile for shadowsocks-rust";
+      };
+
+    systemd.services.shadowsocks-rust = {
+      description = "shadowsocks-rust Daemon";
+      after = [ "network.target" ];
+      wantedBy = [ "multi-user.target" ];
+      path = [ pkgs.shadowsocks-rust ]
+        ++ optional (cfg.plugin != null) cfg.plugin
+        ++ optional (cfg.passwordFile != null) pkgs.jq;
+      serviceConfig.PrivateTmp = true;
+      script = ''
+        ${optionalString (cfg.passwordFile != null) ''
+          cat ${configFile} | jq --arg password "$(cat "${cfg.passwordFile}")" '. + { password: $password }' > /run/shadowsocks.json
+        ''}
+        exec ssserver --config ${if cfg.passwordFile != null then "/run/shadowsocks.json" else configFile}
+      '';
+    };
+  };
+}

nixos/module/generic/shadowsocks.nix

@@ -0,0 +1,28 @@
+{
+  ...
+}:
+{
+  pkgs,
+  config,
+  ...
+}:
+{
+  sops.secrets."ss-bfs/password" = {};
+  services.shadowsocks-rust = {
+    enable = true;
+    plugin = "${pkgs.shadowsocks-v2ray-plugin}/bin/v2ray-plugin";
+    # TODO: setup dnscrypt or a private DNS server for this
+    # extraConfig = {
+    #   nameserver = "185.12.64.1"; # FIXME: this can vary across instances.
+    # };
+    port = 55228;
+    pluginOpts = "server";
+    # TODO: setup a TLS certs for this (look: (README.md) https://github.com/shadowsocks/v2ray-plugin/)
+    #pluginOpts = "server;tls;host=ss.bfs.band";
+    passwordFile = config.sops.secrets."ss-bfs/password".path;
+    mode = "tcp_and_udp"; # default
+    localAddress = "0.0.0.0";
+    fastOpen = true; # default
+    encryptionMethod = "chacha20-ietf-poly1305"; # default
+  };
+}

nixos/module/generic/xray-system.nix

@@ -0,0 +1,101 @@
+{
+  inputs,
+  flake,
+  self,
+}: {
+  lib,
+  pkgs,
+  modulesPath,
+  config,
+  ...
+}: let
+  cfg = config.hectic.generic.xray-system;
+  xrayPort = 10086;
+in {
+  imports = [
+    self.nixosModules.hectic
+    inputs.sops-nix.nixosModules.sops
+  ];
+
+  options.hectic.generic.xray-system = {
+    enable = lib.mkEnableOption "generic xray VPN server system configuration";
+
+    defaultSopsFile = lib.mkOption {
+      type        = lib.types.path;
+      description = ''
+        SOPS-encrypted secrets file used as `sops.defaultSopsFile`.
+        Must define the `config` and `init-postgresql` secrets.
+      '';
+      example     = lib.literalExpression "../../../sus/bfs.xray.yaml";
+    };
+  };
+
+  config = lib.mkIf cfg.enable {
+    services.xray = {
+      enable       = true;
+      settingsFile = config.sops.secrets."config".path;
+    };
+
+    users.users.root.openssh.authorizedKeys.keys = [
+      ''ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIOn1KflaIX1RU9YS/qLb0GInmndYxx2vTLZC9OA+eXZl''
+      ''ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIBKPbIJATVyAw7F7vBZbHkCODXFo5gvDyqhuU0gnNUNH''
+    ];
+
+    boot.initrd.availableKernelModules = [
+      "ata_piix"
+      "uhci_hcd"
+      "xen_blkfront"
+    ] ++ (if pkgs.stdenv.hostPlatform.system != "aarch64-linux" then [ "vmw_pvscsi" ] else []);
+    boot.initrd.kernelModules = ["nvme"];
+
+    disko.devices = {
+      disk.vda = {
+        device  = lib.mkDefault "/dev/vda";
+        content = {
+          type       = "gpt";
+          partitions = {
+            boot = {
+              size     = "1M";
+              type     = "EF02";
+              priority = 1;
+            };
+            root = {
+              size    = "100%";
+              content = {
+                type       = "filesystem";
+                format     = "ext4";
+                mountpoint = "/";
+              };
+            };
+          };
+        };
+      };
+    };
+
+    hectic = {
+      archetype.base.enable = true;
+      archetype.dev.enable  = true;
+    };
+
+    sops = {
+      gnupg.sshKeyPaths = [ ];
+      age.sshKeyPaths   = [ "/etc/ssh/ssh_host_ed25519_key" ];
+      defaultSopsFile   = cfg.defaultSopsFile;
+
+      secrets."config"          = {};
+      secrets."init-postgresql" = {};
+    };
+
+    networking.firewall = {
+      enable          = true;
+      allowedTCPPorts = [
+        xrayPort 8443
+        80 443 # for acme
+      ];
+    };
+
+    environment.systemPackages = with pkgs; [
+      xray
+    ];
+  };
+}

nixos/module/hectic/archetype/base.nix

@@ -0,0 +1,54 @@
+{ 
+  inputs,
+  self,
+  ...
+}: {
+  pkgs,
+  lib,
+  config,
+  ...
+}: let
+  cfg = config.hectic.archetype.base;
+in {
+  imports = [
+    inputs.disko.nixosModules.default
+    inputs.nixos-mailserver.nixosModules.mailserver
+  ];
+
+  options.hectic.archetype.base.enable = lib.mkEnableOption "Enable archetupe.dev";
+
+  config = lib.mkIf cfg.enable {
+    hectic = {
+      program.zsh.enable    = lib.mkDefault true;
+      program.tmux.enable   = lib.mkDefault true;
+      program.nixvim.enable = lib.mkDefault true;
+    };
+
+    users.defaultUserShell = pkgs.zsh;
+
+    # Enable flakes and new 'nix' command
+    nix.settings = {
+      experimental-features = "nix-command flakes";
+      extra-substituters = [
+        "https://cache.hectic-lab.com/hectic"
+      ];
+      extra-trusted-public-keys = [
+        "hectic:KMQsKow4SoA9K2vOJlOljmx7/Zpf91Yy+5qEtxDDCzA="
+      ];
+    };
+
+    networking.firewall.enable = true;
+
+    environment = {
+      defaultPackages = [];
+      systemPackages = (with self.packages.${pkgs.stdenv.hostPlatform.system}; [
+        nvim-pager
+      ]);
+      variables = {
+        PAGER = with self.packages.${pkgs.stdenv.hostPlatform.system}; "${nvim-pager}/bin/pager";
+      };
+    };
+
+    system.stateVersion = "25.05";
+  };
+}

nixos/module/hectic/archetype/common.nix

@@ -0,0 +1 @@
+{ ... }: {}

nixos/module/hectic/archetype/dev.nix

@@ -0,0 +1,70 @@
+{ 
+  inputs,
+  flake,
+  self,
+}: {
+  pkgs,
+  modulesPath,
+  lib,
+  config,
+  ...
+}: let
+  cfg = config.hectic.archetype.dev;
+in {
+  # necessary imports:
+  imports = [
+    (modulesPath + "/profiles/qemu-guest.nix")
+  ];
+
+  options.hectic.archetype.dev.enable = lib.mkEnableOption "Enable archetupe.dev";
+
+  config = lib.mkIf cfg.enable {
+    hectic.archetype.base.enable = true;
+
+    services.getty.autologinUser = "root";
+
+    virtualisation.vmVariant.virtualisation = {
+      qemu.options = [
+        "-nographic"
+        "-display curses"
+        "-append console=ttyS0"
+        "-serial mon:stdio"
+        "-vga qxl"
+      ];
+      forwardPorts = [
+        {
+          from = "host";
+          host.port = 40500;
+          guest.port = 22;
+        }
+      ];
+    };
+
+    services.openssh = {
+      enable = true;
+      settings = {
+        PasswordAuthentication = false;
+      };
+    };
+
+    environment = {
+      systemPackages =
+        (with pkgs; [
+          curl
+          neovim
+          yq-go
+          jq
+          htop-vim
+        ]);
+    };
+
+    # Adjust zsh prompt for dev archetype: show '#' instead of '%'
+    home-manager.sharedModules = lib.mkAfter [
+      {
+        programs.zsh.initContent = lib.mkAfter ''
+          PROMPT="# %~ "
+        '';
+      }
+    ];
+  };
+}

nixos/module/hectic/archetype/explosive.nix

@@ -0,0 +1,3 @@
+{ ... }: { lib, ... }: {
+  options.hectic.archetype.explosive.enable = lib.mkEnableOption "Enable impermanence usage";
+}

nixos/module/hectic/hardware/cloudzy.nix

@@ -0,0 +1,117 @@
+{ 
+  inputs,
+  flake,
+  self,
+}:
+{ 
+  pkgs,
+  lib,
+  config,
+  ...
+}: let
+  cfg = config.hectic.hardware.cloudzy;
+in {
+  options.hectic.hardware.cloudzy = {
+    enable = lib.mkEnableOption "Enable hetzner-cloud hardware configurations";
+    ipGateway = lib.mkOption {
+      type        = lib.types.strMatching "^([0-9]{1,3}\\.){3}[0-9]{1,3}$";
+      example     = "188.243.124.1";
+      description = ''
+        
+      '';
+    };
+    ipv4 = lib.mkOption {
+      type        = lib.types.strMatching "^([0-9]{1,3}\\.){3}[0-9]{1,3}$";
+      example     = "188.243.124.246";
+      description = ''
+        
+      '';
+    };
+    prefixLength = lib.mkOption {
+      type        = lib.types.int;
+      example     = 24;
+      description = ''
+        
+      '';
+    };
+    device = lib.mkOption {
+      type = lib.types.str;
+      default = "/dev/sda";
+      example = "/dev/disk/by-uuid/f184a16b-6eca-41cb-b48a-ff37cdce1d79";
+      description = ''
+        boot device uuid 
+	if it is null then will use "/dev/sda" 
+	/dev/sda - default hetzner cloud device
+	!! But can changes on reboot if server have volumes
+	!! So use IDs
+      '';
+    };
+    networkMatchConfigName = lib.mkOption {
+      type = lib.types.str;
+      example = "enp1s0";
+      description = ''
+        type of network conection, 
+	on older hetzner servers may be `ens3`
+        on newer probably `enp1s0`
+
+	you can use `networkctl list` on server to know it
+      '';
+    };
+  };
+
+  config = lib.mkIf cfg.enable {
+    boot.loader.systemd-boot.enable = false;
+    boot.loader.efi.canTouchEfiVariables = false;
+    
+    boot.loader.grub = {
+      enable = true;
+      device = cfg.device;
+      efiSupport = false;
+      forceInstall = true;
+    };
+
+    disko.devices.disk.main = {
+      device = cfg.device;
+      type = "disk";
+      content = {
+        type = "gpt";
+        partitions = {
+          ESP = {
+            size = "512M";
+            type = "EF00";
+            content = {
+              type = "filesystem";
+              format = "vfat";
+              mountpoint = "/boot";
+              mountOptions = [ "umask=0077" ];
+            };
+          };
+          root = {
+            size = "100%";
+            content = {
+              type = "filesystem";
+              format = "ext4";
+              mountpoint = "/";
+            };
+          };
+        };
+      };
+    };
+
+    networking.useDHCP = false;
+    networking.interfaces."30-wan" = {
+      matchConfig.Name = cfg.networkMatchConfigName;
+      ipv4.addresses = [
+        { address = cfg.ipv4; prefixLength = cfg.prefixLength; }
+      ];
+    };
+    networking.defaultGateway = cfg.ipGateway;
+    networking.nameservers = [ "1.1.1.1" "8.8.8.8" ];
+
+    boot.initrd.availableKernelModules = [
+      "ata_piix"
+      "uhci_hcd"
+      "xen_blkfront"
+    ] ++ (if pkgs.stdenv.hostPlatform.system != "aarch64-linux" then [ "vmw_pvscsi" ] else []);
+  };
+}

nixos/module/hectic/hardware/geo-hosting.nix

@@ -0,0 +1,102 @@
+{ 
+  inputs,
+  flake,
+  self,
+}:
+{ 
+  pkgs,
+  lib,
+  config,
+  ...
+}: let
+  cfg = config.hectic.hardware.geo-hosting;
+in {
+  options.hectic.hardware.geo-hosting = {
+    enable = lib.mkEnableOption "Enable geo-hosting hardware configurations";
+    ipv4Gateway = lib.mkOption {
+      type        = lib.types.strMatching "^([0-9]{1,3}\\.){3}[0-9]{1,3}$";
+      example     = "188.243.124.1";
+      description = ''
+        
+      '';
+    };
+    ipv4 = lib.mkOption {
+      type        = lib.types.strMatching "^([0-9]{1,3}\\.){3}[0-9]{1,3}$";
+      example     = "188.243.124.246";
+      description = ''
+        
+      '';
+    };
+    device = lib.mkOption {
+      type = lib.types.str;
+      default = "/dev/vda";
+      example = "/dev/disk/by-uuid/f184a16b-6eca-41cb-b48a-ff37cdce1d79";
+      description = ''
+        boot device uuid 
+        if it is null then will use "/dev/vda" 
+        /dev/sva - default geo hosting device
+        !! But can changes on reboot if server have volumes
+        !! So use IDs
+      '';
+    };
+    networkMatchConfigName = lib.mkOption {
+      type = lib.types.strMatching "^(enp1s0|ens3)$";
+      example = "ens3";
+      description = ''
+        type of network conection
+
+        you can use `networkctl list` on server to know it
+      '';
+    };
+  };
+
+  config = lib.mkIf cfg.enable {
+    boot.loader.systemd-boot.enable = false;
+    boot.loader.efi.canTouchEfiVariables = false;
+    
+    boot.loader.grub = {
+      enable = true;
+      device = cfg.device;
+      efiSupport = false;
+      forceInstall = true;
+    };
+
+    disko.devices.disk.vda = {
+      device = cfg.device;
+      type = "disk";
+      content = {
+        type = "gpt";
+        partitions = {
+          ESP = {
+            size = "512M";
+            type = "EF00";
+            content = {
+              type = "filesystem";
+              format = "vfat";
+              mountpoint = "/boot";
+              mountOptions = [ "umask=0077" ];
+            };
+          };
+          root = {
+            size = "100%";
+            content = {
+              type = "filesystem";
+              format = "ext4";
+              mountpoint = "/";
+            };
+          };
+        };
+      };
+    };
+
+
+    networking.useDHCP = false;
+    networking.interfaces.${cfg.networkMatchConfigName} = {
+      ipv4.addresses = [
+        { address = cfg.ipv4; prefixLength = 24; }
+      ];
+    };
+    networking.defaultGateway = cfg.ipv4Gateway;
+    networking.nameservers = [ "1.1.1.1" "8.8.8.8" ];
+  };
+}

nixos/module/hectic/hardware/hetzner-cloud.nix

@@ -0,0 +1,188 @@
+{
+  ...
+}:
+{ 
+  pkgs,
+  lib,
+  config,
+  ...
+}: let
+  cfg = config.hectic.hardware.hetzner-cloud;
+  isNewer = cfg.generation == "newer";
+  networkMatchConfig =
+    (lib.optionalAttrs (cfg.networkMatchConfigName != null) {
+      Name = cfg.networkMatchConfigName;
+    })
+    // (lib.optionalAttrs (cfg.networkMatchConfigMac != null) {
+      PermanentMACAddress = cfg.networkMatchConfigMac;
+    });
+in {
+  options.hectic.hardware.hetzner-cloud = {
+    enable = lib.mkEnableOption "Enable hetzner-cloud hardware configurations";
+    generation = lib.mkOption {
+      type = lib.types.enum [ "classic" "newer" ];
+      default = "classic";
+      description = ''
+        Hetzner server generation profile.
+
+        `classic` keeps the historical `/dev/sda` assumption.
+        `newer` is for ccx/NVMe-era servers and defaults the disk device to
+        `/dev/nvme0n1`.
+      '';
+    };
+    #bootParUuid = lib.mkOption {
+    #  type = with lib.types; nullOr oneOf [
+    #    (lib.types.strMatching "^[0-9A-Fa-f]{4}-[0-9A-Fa-f]{4}$")
+    #    (lib.types.strMatching "^[0-9a-fA-F-]{36}$")
+    #  ];
+    #  default = null;
+    #  example = "5628-19B6";
+    #  description = ''
+    #    boot partition uuid if it is null 
+    #    then will use "/dev/sda15" (default hetzner cloud boot device)
+    #  '';
+    #};
+    ipv4 = lib.mkOption {
+      type        = lib.types.strMatching "^([0-9]{1,3}\\.){3}[0-9]{1,3}$";
+      example     = "188.243.124.246";
+      description = ''
+        
+      '';
+    };
+    ipv6 = lib.mkOption {
+      type        = lib.types.strMatching "^([0-9a-fA-F]{1,4}:){3}[0-9a-fA-F]{1,4}$";
+      example     = "2a01:4f8:1c1a:d883";
+      description = ''
+        
+      '';
+    };
+    floatingIpv4 = lib.mkOption {
+      type = with lib.types; nullOr (strMatching "^([0-9]{1,3}\\.){3}[0-9]{1,3}$");
+      default = null;
+      example = "188.243.124.247";
+      description = ''
+        Optional Hetzner Floating IPv4 configured as `/32` on the primary interface.
+      '';
+    };
+    device = lib.mkOption {
+      type = lib.types.str;
+      default = if isNewer then "/dev/nvme0n1" else "/dev/sda";
+      example = "/dev/disk/by-uuid/f184a16b-6eca-41cb-b48a-ff37cdce1d79";
+      description = ''
+        boot device uuid 
+        if it is null then will use "/dev/sda" 
+        /dev/sda - default hetzner cloud device
+        /dev/nvme0n1 - default for newer Hetzner generations
+        !! But can changes on reboot if server have volumes
+        !! So use IDs
+      '';
+    };
+    networkMatchConfigName = lib.mkOption {
+      type = with lib.types; nullOr str;
+      default = null;
+      example = "enp1s0";
+      description = ''
+        Optional interface name to match in systemd-networkd.
+
+        Prefer `networkMatchConfigMac` for stable matching across rescue
+        images and installed systems that may rename interfaces differently.
+
+        You can use `networkctl list` on server to know it.
+      '';
+    };
+    networkMatchConfigMac = lib.mkOption {
+      type = with lib.types; nullOr (strMatching "^([0-9A-Fa-f]{2}:){5}[0-9A-Fa-f]{2}$");
+      default = null;
+      example = "92:00:08:4a:b0:32";
+      description = ''
+        Optional permanent MAC address to match in systemd-networkd.
+
+        This is the preferred Hetzner Cloud matching method because interface
+        names can differ between rescue images and installed NixOS systems.
+      '';
+    };
+  };
+
+  config = lib.mkIf cfg.enable (lib.mkMerge
+  [
+    {
+      boot.loader.systemd-boot.enable = false;
+      boot.loader.efi.canTouchEfiVariables = false;
+      boot.loader.grub = {
+        enable = true;
+        efiSupport = true;
+        efiInstallAsRemovable = true;
+        device = "nodev";
+      };
+
+      boot.initrd.availableKernelModules = [
+        "ata_piix"
+        "uhci_hcd"
+        "xen_blkfront"
+      ] ++ (if pkgs.stdenv.hostPlatform.system != "aarch64-linux" then [ "vmw_pvscsi" ] else []);
+
+      networking.useDHCP    = false;
+      networking.useNetworkd = true;
+      systemd.network.enable = true;
+      systemd.network.networks."30-wan" = {
+        matchConfig = networkMatchConfig;
+        networkConfig.DHCP = "no";
+        address = [
+          "${cfg.ipv4}/32"
+          "${cfg.ipv6}::/64"
+        ] ++ lib.optional (cfg.floatingIpv4 != null) "${cfg.floatingIpv4}/32";
+        routes = [
+          { Gateway = "172.31.1.1"; GatewayOnLink = true; }
+          { Gateway = "fe80::1"; }
+        ];
+      };
+
+      disko.devices = {
+        disk = {
+          main = {
+            type = "disk";
+            device = cfg.device;
+            content = {
+              type = "gpt";
+              partitions = {
+                boot = {
+                  size = "1M";
+                  type = "EF02";
+                  priority = 1;
+                };
+                ESP = {
+                  size = "512M";
+                  type = "EF00";
+                  content = {
+                    type = "filesystem";
+                    format = "vfat";
+                    mountpoint = "/boot";
+                  };
+                };
+                root = {
+                  size = "100%";
+                  content = {
+                    type = "filesystem";
+                    format = "ext4";
+                    mountpoint = "/";
+                  };
+                };
+              };
+            };
+          };
+        };
+      };
+
+      assertions = [
+        {
+          assertion = cfg.networkMatchConfigName != null || cfg.networkMatchConfigMac != null;
+          message = "hectic.hardware.hetzner-cloud requires networkMatchConfigName or networkMatchConfigMac";
+        }
+      ];
+    } 
+    (lib.mkIf (pkgs.stdenv.hostPlatform.system == "aarch64-linux") {
+      boot.initrd.kernelModules = [ "virtio_gpu" ];
+      boot.kernelParams = [ "console=tty" ];
+    })
+  ]);
+}

nixos/module/hectic/hardware/lenovo-ideapad-15arh7.nix

@@ -0,0 +1,227 @@
+{
+  inputs,
+  ...
+}:
+{ 
+  lib,
+  config,
+  modulesPath,
+  pkgs,
+  ...
+}: let
+  cfg = config.hectic.hardware.lenovo-ideapad-15arh7;
+  hasDisko = false;
+in {
+  # FIXME: FUCK
+  #imports = [
+  #  "${inputs.nixos-hardware}/common/cpu/amd"
+  #  "${inputs.nixos-hardware}/common/cpu/amd/pstate.nix"
+  #  "${inputs.nixos-hardware}/common/gpu/amd"
+  #  "${inputs.nixos-hardware}/common/gpu/nvidia/prime-sync.nix"
+  #  "${inputs.nixos-hardware}/common/pc/laptop"
+  #  "${inputs.nixos-hardware}/common/pc/laptop/ssd"
+  #];
+
+  options.hectic.hardware.lenovo-ideapad-15arh7 = {
+    enable = lib.mkEnableOption "Enable lenovo-legion hardware configurations";
+    swapSize = lib.mkOption {
+      type = lib.types.either (lib.types.enum [ "100%" ]) (lib.types.strMatching "[0-9]+[KMGTP]?");
+      default = "0";
+      description = ''
+        Size of the partition, in sgdisk format.
+        sets end automatically with the + prefix
+        can be 100% for the whole remaining disk, will be done last in that case.
+      '';
+    };
+    device = lib.mkOption {
+      type = lib.types.str;
+      default = "0";
+      description = ''
+        Size of the partition, in sgdisk format.
+        sets end automatically with the + prefix
+        can be 100% for the whole remaining disk, will be done last in that case.
+      '';
+    };
+  };
+
+  config = lib.mkIf cfg.enable {
+    /* common */
+    hardware.nvidia = {
+      modesetting.enable = true;
+      prime = {
+        amdgpuBusId = "PCI:5:0:0";
+        nvidiaBusId = "PCI:1:0:0";
+      };
+    };
+    
+    environment.systemPackages = with pkgs; [
+      vulkan-tools
+    ];
+    /* */
+
+    /* boot */
+    boot.initrd.availableKernelModules = [
+      "nvme"
+      "xhci_pci"
+      "usb_storage"
+      "usbhid"
+      "sd_mod"
+    ];
+    boot.initrd.kernelModules = [ "dm-snapshot" "amdgpu" ];
+    boot.kernelModules = [ "kvm-amd" ];
+    boot.extraModulePackages = [ ];
+    /* */
+
+    networking.useDHCP = lib.mkDefault true;
+    nixpkgs.hostPlatform = lib.mkDefault "x86_64-linux";
+
+    /* cpu */
+    hardware.cpu.amd.updateMicrocode = lib.mkDefault config.hardware.enableRedistributableFirmware;
+
+    /* gpu */
+    services.xserver.videoDrivers = [ 
+      "nvidia" 
+      #"amdgpu"  # NOTE: probably useles with nvidia optimus prime
+      #"nouveau" # NOTE: open source nvidia
+    ];
+
+    hardware.opengl = {
+      enable = true;
+      driSupport32Bit = true;
+      extraPackages = with pkgs; [
+        vulkan-loader
+        vulkan-validation-layers
+        vulkan-extension-layer
+        amdvlk
+
+      ];
+      extraPackages32 = with pkgs; [
+        pkgsi686Linux.vulkan-loader
+        pkgsi686Linux.vulkan-validation-layers
+        pkgsi686Linux.vulkan-extension-layer
+        driversi686Linux.amdvlk
+      ];
+    };
+
+    #environment.variables.VK_DRIVER_FILES=/run/opengl-driver/share/vulkan/icd.d/nvidia_icd.x86_64.json;
+    #environment.sessionVariables.VK_DRIVER_FILES = "/run/opengl-driver/share/vulkan/icd.d/nvidia_icd.x86_64.json";
+
+    #environment.sessionVariables = rec {
+    #  VK_ICD_FILENAMES = 
+    #    "${config.hardware.nvidia.package}/share/vulkan/icd.d/nvidia_icd.x86_64.json";
+
+    #    #:${config.environment.variables.VK_ICD_FILENAMES or ""}";
+    #};
+
+
+    hardware.nvidia = {
+      # Nvidia power management. Experimental, and can cause sleep/suspend to fail.
+      # Enable this if you have graphical corruption issues or application crashes after waking
+      # up from sleep. This fixes it by saving the entire VRAM memory to /tmp/ instead 
+      # of just the bare essentials.
+      powerManagement.enable = false;
+
+      # Fine-grained power management. Turns off GPU when not in use.
+      # Experimental and only works on modern Nvidia GPUs (Turing or newer).
+      powerManagement.finegrained = false;
+
+      # Use the NVidia open source kernel module (not to be confused with the
+      # independent third-party "nouveau" open source driver).
+      # Support is limited to the Turing and later architectures. Full list of 
+      # supported GPUs is at: 
+      # https://github.com/NVIDIA/open-gpu-kernel-modules#compatible-gpus 
+      # Only available from driver 515.43.04+
+      # Currently alpha-quality/buggy, so false is currently the recommended setting.
+      open = false;
+
+      # Enable the Nvidia settings menu,
+      # accessible via `nvidia-settings`.
+      nvidiaSettings = true;
+
+      # nvidia package overwrive
+      package = config.boot.kernelPackages.nvidiaPackages.stable;
+
+    };
+    /* */
+
+    /* sound */
+    hardware.pulseaudio.enable = true;
+    hardware.pulseaudio.support32Bit = true;
+    /* */
+
+    /* disk */
+    disko.devices = {
+      disk.main = {
+        inherit (cfg) device;
+        type = "disk";
+        content = {
+          type = "gpt";
+          partitions = {
+            boot = {
+              name = "boot";
+              size = "1M";
+              type = "EF02";
+            };
+            esp = {
+              name = "ESP";
+              size = "500M";
+              type = "EF00";
+              content = {
+                type = "filesystem";
+                format = "vfat";
+                mountpoint = "/boot";
+              };
+            };
+            swap = {
+              size = cfg.swapSize;
+              content = {
+                type = "swap";
+                resumeDevice = true;
+              };
+            };
+            root = {
+              name = "root";
+              size = "100%";
+              content = {
+                type = "lvm_pv";
+                vg = "root_vg";
+              };
+            };
+          };
+        };
+      };
+      lvm_vg = {
+        root_vg = {
+          type = "lvm_vg";
+          lvs = {
+            root = {
+              size = "100%FREE";
+              content = {
+                type = "btrfs";
+                extraArgs = ["-f"];
+
+                subvolumes = lib.mkMerge [
+		  {
+                    "/root" = {
+                      mountpoint = "/";
+                    };
+                    "/nix" = {
+                      mountOptions = ["subvol=nix" "noatime"];
+                      mountpoint = "/nix";
+                    };
+		  }
+		  (if config.hectic.archetype.explosive.enable then {
+                    "/persist" = {
+                      mountOptions = ["subvol=persist" "noatime"];
+                      mountpoint = "/persist";
+                    };
+		  } else {})
+		];
+              };
+            };
+          };
+        };
+      };
+    };
+  };
+}

nixos/module/hectic/hardware/njalla.nix

@@ -0,0 +1,134 @@
+{ ... }:
+{ 
+  lib,
+  config,
+  ...
+}: let
+  cfg = config.hectic.hardware.njalla;
+in {
+  options.hectic.hardware.njalla = {
+    enable = lib.mkEnableOption "Enable njalla hardware configurations";
+    ipv4 = lib.mkOption {
+      type        = lib.types.strMatching "^([0-9]{1,3}\\.){3}[0-9]{1,3}$";
+      example     = "185.193.126.103";
+      description = ''
+        Njalla IPv4 address assigned to the host.
+      '';
+    };
+    ipv4PrefixLength = lib.mkOption {
+      type        = lib.types.int;
+      default     = 24;
+      example     = 24;
+      description = ''
+        Njalla IPv4 prefix length.
+      '';
+    };
+    ipv4Gateway = lib.mkOption {
+      type        = lib.types.strMatching "^([0-9]{1,3}\\.){3}[0-9]{1,3}$";
+      default     = "185.193.126.1";
+      example     = "185.193.126.1";
+      description = ''
+        Njalla IPv4 gateway.
+      '';
+    };
+    ipv6 = lib.mkOption {
+      type        = lib.types.strMatching "^([0-9a-fA-F]{0,4}:){2,7}[0-9a-fA-F]{0,4}$";
+      example     = "2a0a:3840:1337:126:0:b9c1:7e67:1337";
+      description = ''
+        Njalla IPv6 address assigned to the host.
+      '';
+    };
+    ipv6PrefixLength = lib.mkOption {
+      type        = lib.types.int;
+      default     = 64;
+      example     = 64;
+      description = ''
+        Njalla IPv6 prefix length.
+      '';
+    };
+    ipv6Gateway = lib.mkOption {
+      type        = lib.types.strMatching "^([0-9a-fA-F]{0,4}:){2,7}[0-9a-fA-F]{0,4}$";
+      default     = "2a0a:3840:1337:126::1";
+      example     = "2a0a:3840:1337:126::1";
+      description = ''
+        Njalla IPv6 gateway.
+      '';
+    };
+    networkMatchConfigName = lib.mkOption {
+      type        = lib.types.str;
+      default     = "eth0";
+      example     = "eth0";
+      description = ''
+        Njalla container network interface name.
+      '';
+    };
+    device = lib.mkOption {
+      type = lib.types.str;
+      default = "/dev/vda";
+      example = "/dev/disk/by-id/virtio-root";
+      description = ''
+        Njalla installation disk for disko/nixos-anywhere.
+
+        `/dev/vda` is the default block device visible on the inspected Njalla
+        host. Prefer a stable `/dev/disk/by-id/...` path when available.
+      '';
+    };
+    enableDisko = lib.mkOption {
+      type        = lib.types.bool;
+      default     = true;
+      description = ''
+        Whether to provide a disko layout for nixos-anywhere installs.
+      '';
+    };
+  };
+
+  config = lib.mkIf cfg.enable (lib.mkMerge [
+    {
+      boot.isContainer = true;
+
+      networking.useDHCP    = false;
+      networking.useNetworkd = true;
+      systemd.network.enable = true;
+      systemd.network.networks."30-wan" = {
+        matchConfig.Name = cfg.networkMatchConfigName;
+        networkConfig.DHCP = "no";
+        address = [
+          "${cfg.ipv4}/${toString cfg.ipv4PrefixLength}"
+          "${cfg.ipv6}/${toString cfg.ipv6PrefixLength}"
+        ];
+        routes = [
+          { Gateway = cfg.ipv4Gateway; }
+          { Gateway = cfg.ipv6Gateway; }
+        ];
+      };
+
+      networking.nameservers = [ "1.1.1.1" "8.8.8.8" ];
+    }
+    (lib.mkIf cfg.enableDisko {
+      boot.loader.grub.device = cfg.device;
+
+      disko.devices.disk.main = {
+        type = "disk";
+        device = cfg.device;
+        content = {
+          type = "gpt";
+          partitions = {
+            boot = {
+              size     = "1M";
+              type     = "EF02";
+              priority = 1;
+            };
+            root = {
+              size    = "100%";
+              content = {
+                type       = "filesystem";
+                format     = "ext4";
+                mountpoint = "/";
+              };
+            };
+          };
+        };
+      };
+    })
+  ]);
+}

nixos/module/hectic/hardware/zomro.nix

@@ -0,0 +1,70 @@
+{ 
+  inputs,
+  flake,
+  self,
+}:
+{ 
+  pkgs,
+  lib,
+  config,
+  ...
+}: let
+  cfg = config.hectic.hardware.zomro;
+in {
+  options.hectic.hardware.zomro = {
+    enable = lib.mkEnableOption "Enable zomro hardware configurations";
+    device = lib.mkOption {
+      type = lib.types.str;
+      default = "/dev/vda";
+      example = "/dev/disk/by-uuid/f184a16b-6eca-41cb-b48a-ff37cdce1d79";
+      description = ''
+        boot device uuid 
+        if it is null then will use "/dev/vda" 
+        /dev/vda - default zomro device
+        !! But can changes on reboot if server have volumes
+        !! So use IDs
+      '';
+    };
+  };
+
+  config = lib.mkIf cfg.enable (lib.mkMerge
+  [
+    {
+      boot.loader.grub.device =  cfg.device;
+      boot.initrd.availableKernelModules = [
+        "ata_piix"
+        "uhci_hcd"
+        "xen_blkfront"
+      ] ++ (if pkgs.stdenv.hostPlatform.system != "aarch64-linux" then [ "vmw_pvscsi" ] else []);
+      boot.initrd.kernelModules = ["nvme"];
+
+      disko.devices = {
+        disk.master = {
+          device = cfg.device;
+          content = {
+            type = "gpt";
+            partitions = {
+              boot = {
+                size     = "1M";
+                type     = "EF02";
+                priority = 1;
+              };
+              root = {
+                size    = "100%";
+                content = {
+                  type       = "filesystem";
+                  format     = "ext4";
+                  mountpoint = "/";
+                };
+              };
+            };
+          };
+        };
+      };
+    } 
+    (lib.mkIf (pkgs.stdenv.hostPlatform.system == "aarch64-linux") {
+      boot.initrd.kernelModules = [ "virtio_gpu" ];
+      boot.kernelParams = [ "console=tty" ];
+    })
+  ]);
+}

nixos/module/hectic/program/nixvim.nix

@@ -0,0 +1,197 @@
+{
+  inputs,
+  flake,
+  self,
+}: {
+  pkgs,
+  lib,
+  config,
+  ...
+}: let
+  cfg = config.hectic.program.nixvim;
+in {
+  imports = [
+    inputs.nixvim.nixosModules.nixvim
+  ];
+
+  options.hectic.program.nixvim.enable = lib.mkEnableOption "Enable hectic nixvim config";
+
+  config = lib.mkIf cfg.enable {
+    programs.nixvim = {
+      enable = true;
+
+      extraPackages = with pkgs; [ gcc ];
+
+      colorschemes.kanagawa = {
+        enable = true;
+        settings.colors.theme.all = {};
+      };
+
+      opts = {
+        spell      = true;
+        spelllang  = [ "en" "ru" "it" ];
+        tabstop    = 2;
+        shiftwidth = 2;
+        softtabstop = 2;
+        expandtab  = true;
+      };
+
+      extraFiles = {
+        "spell/ru.utf-8.spl".source = pkgs.fetchurl {
+          url    = "https://ftp.nluug.nl/vim/runtime/spell/ru.utf-8.spl";
+          sha256 = "sha256-6y0714ogILMLzAp8/r2s6/t6QnWBEU9muIpXeubaxU0=";
+        };
+        "spell/ru.utf-8.sug".source = pkgs.fetchurl {
+          url    = "https://ftp.nluug.nl/vim/runtime/spell/ru.utf-8.sug";
+          sha256 = "sha256-6r2GForYXVv7gGiAjPeYK6sDdK/CmctJ7MidcWFvOTs=";
+        };
+        "spell/it.utf-8.spl".source = pkgs.fetchurl {
+          url  = "https://ftp.nluug.nl/vim/runtime/spell/it.utf-8.spl";
+          hash = "sha256-2AczkD6DbVN5DAq4wcLyn2Y8oqd67ns4Guprh2KudBM=";
+        };
+        "spell/it.utf-8.sug".source = pkgs.fetchurl {
+          url  = "https://ftp.nluug.nl/vim/runtime/spell/it.utf-8.sug";
+          hash = "sha256-4LsXYaeScJJrdaj69PTQ2EDVWis0UY/Y5RKSfCckzko=";
+        };
+        "ftdetect/hemar.vim".text = ''
+          au BufRead,BufNewFile *.hemar setfiletype hemar
+        '';
+        "queries/hemar/highlights.scm".text = ''
+          (interpolation)      @keyword
+
+          (for "for"   @keyword)
+          (for "in"    @keyword)
+          (done "done" @keyword)
+
+          (path)   @field
+          (string) @string
+          (text)   @text
+
+          (for
+            "{[" @punctuation.bracket
+            "]}" @punctuation.bracket)
+
+          (done
+            "{[" @punctuation.bracket
+            "]}" @punctuation.bracket)
+
+          (interpolation
+            "{[" @punctuation.bracket
+            "]}" @punctuation.bracket)
+        '';
+      };
+
+      extraConfigLuaPre = /* lua */ ''
+        -- map leader
+        vim.api.nvim_set_keymap("", "<Space>", "<Nop>", { noremap = true, silent = true })
+        vim.g.mapleader = ' '
+
+        -- render markdown
+        require('render-markdown').setup({
+          link = {
+            enabled = true,
+            render_modes = false,
+          },
+        })
+
+        -- nowrap for *.nowrap.* markdown files
+        vim.api.nvim_create_autocmd("FileType", {
+          pattern = "markdown",
+          callback = function()
+            if vim.fn.expand("%:t"):find("%.nowrap%.") then vim.opt_local.wrap = false end
+          end,
+        })
+
+        -- toggle conceallevel
+        vim.keymap.set("n", "<leader>tc", ":setlocal <C-R>=&conceallevel ? 'conceallevel=0' : 'conceallevel=2'<CR><CR>", { desc = "[T]oggle [C]onceallevel" })
+
+        -- tree-sitter: register hemar parser
+        local parser_config = require("nvim-treesitter.parsers").get_parser_configs()
+        parser_config.hemar = {
+          install_info = {
+            url   = "https://github.com/hectic-lab/util.nix",
+            files = { "package/hemar/grammar/tree-sitter/src/parser.c" },
+            generate_requires_npm      = false,
+            requires_generate_from_grammar = false,
+          },
+          filetype = "hemar",
+        }
+      '';
+
+      extraConfigLuaPost = /* lua */ ''
+        vim.cmd [[
+          hi Normal   guibg=none ctermbg=none
+          hi NonText  guibg=none ctermbg=none
+        ]]
+      '';
+
+      keymaps = [
+        { mode = "n"; key = "<leader>o";  options.silent = true; action = "<cmd>Oil<CR>"; }
+        { mode = "n"; key = "<leader>dd"; action = "<cmd>lua vim.diagnostic.open_float()<CR>"; }
+        { mode = "n"; key = "<leader>dn"; action = "<cmd>lua vim.diagnostic.goto_next()<CR>"; }
+        { mode = "n"; key = "<leader>dp"; action = "<cmd>lua vim.diagnostic.goto_prev()<CR>"; }
+      ];
+
+      extraPlugins = with pkgs.vimPlugins; [
+        nvim-treesitter-parsers.templ
+        vim-shellcheck
+        vim-grammarous
+      ];
+
+      plugins = {
+        render-markdown.enable = true;
+        fidget.enable          = true;
+        oil.enable             = true;
+
+        treesitter = {
+          enable = true;
+          settings = {
+            ensure_installed   = [ "hemar" ];
+            highlight.enable   = true;
+          };
+        };
+
+        lsp = {
+          enable = true;
+          keymaps.lspBuf = {
+            "<leader>lh" = "hover";
+            "<leader>ld" = "definition";
+            "<leader>lD" = "references";
+            "<leader>lr" = "rename";
+            "<leader>li" = "implementation";
+            "<leader>lt" = "type_definition";
+            "<leader>lf" = "format";
+            "<leader>la" = "code_action";
+          };
+          servers = {
+            rust_analyzer = {
+              enable        = true;
+              installRustc  = false;
+              installCargo  = false;
+            };
+            nixd = {
+              enable = true;
+            };
+            nil_ls = {
+              enable        = true;
+              extraOptions.formatting.command = [ "nixpkgs-fmt" ];
+            };
+            clangd.enable                    = true;
+            ts_ls.enable                     = true;
+            gopls.enable                     = true;
+            templ.enable                     = true;
+            bashls.enable                    = true;
+            kotlin_language_server.enable    = true;
+            metals = {
+              enable = true;
+              cmd    = [ "metals" ];
+            };
+            sqls.enable                      = true;
+            java_language_server.enable      = true;
+            pyright.enable                   = true;
+          };
+        };
+      };
+    };
+  };
+}

nixos/module/hectic/program/tmux.nix

@@ -0,0 +1,33 @@
+{
+  inputs,
+  flake,
+  self,
+}: {
+  pkgs,
+  lib,
+  config,
+  ...
+}: let
+  cfg = config.hectic.program.tmux;
+in {
+  imports = [
+    inputs.home-manager.nixosModules.home-manager
+  ];
+
+  options.hectic.program.tmux.enable = lib.mkEnableOption "Enable hectic tmux config";
+
+  config = lib.mkIf cfg.enable {
+    programs.tmux.enable   = true;
+    programs.tmux.terminal = lib.mkOverride 50 "tmux-256color";
+
+    # alias depends on newSession = true (auto-creates session on attach)
+    programs.zsh.shellAliases.tmux = "tmux a";
+    programs.bash.shellAliases.tmux = "tmux a";
+
+    home-manager.sharedModules = [
+      (flake + "/home/module/program/tmux.nix")
+    ];
+
+    home-manager.users.root.home.stateVersion = lib.mkDefault "25.05";
+  };
+}

nixos/module/hectic/program/zsh.nix

@@ -0,0 +1,55 @@
+{
+  inputs,
+  flake,
+  self,
+}: {
+  pkgs,
+  lib,
+  config,
+  ...
+}: let
+  cfg = config.hectic.program.zsh;
+in {
+  imports = [
+    inputs.home-manager.nixosModules.home-manager
+  ];
+
+  options.hectic.program.zsh.enable = lib.mkEnableOption "Enable hectic zsh config";
+
+  config = lib.mkIf cfg.enable {
+    # system-level zsh must be on for home-manager zsh to work
+    programs.zsh.enable = true;
+    users.defaultUserShell = pkgs.zsh;
+
+    # Share the same zsh config with all home-manager users
+    home-manager.sharedModules = [
+      {
+        programs.zsh = {
+          enable                = true;
+          enableCompletion      = true;
+          autosuggestion.enable = true;
+          syntaxHighlighting.enable = true;
+
+          history = {
+            size = 10000;
+            path = "$HOME/.zsh/.zsh_history";
+          };
+
+          oh-my-zsh = {
+            enable = true;
+            theme  = "terminalparty";
+          };
+
+          shellAliases = self.lib.sharedShellAliases;
+
+          initContent = ''
+            set -ovi
+          '';
+        };
+      }
+    ];
+
+    # Still define root for stateVersion; config comes from sharedModules
+    home-manager.users.root.home.stateVersion = lib.mkDefault "25.05";
+  };
+}

nixos/module/hectic/service/astroneer.nix

@@ -0,0 +1,11 @@
+{ 
+  inputs,
+  flake,
+  self,
+}:
+{ 
+  pkgs,
+  lib,
+  config,
+  ...
+}: let in { }

nixos/module/hectic/service/attic.nix

@@ -0,0 +1,73 @@
+{ ... }: {
+  lib,
+  config,
+  ...
+}: let
+  cfg = config.hectic.services.attic;
+in {
+  options.hectic.services.attic = {
+    enable = lib.mkEnableOption "Attic binary cache server";
+
+    hostName = lib.mkOption {
+      type = lib.types.str;
+      description = "Public hostname used by clients to reach this Attic server.";
+    };
+
+    listenAddress = lib.mkOption {
+      type = lib.types.str;
+      default = "127.0.0.1";
+      description = "Local address atticd binds to behind the reverse proxy.";
+    };
+
+    port = lib.mkOption {
+      type = lib.types.port;
+      default = 8080;
+      description = "Local port atticd binds to behind the reverse proxy.";
+    };
+
+    environmentFile = lib.mkOption {
+      type = lib.types.path;
+      description = ''
+        SOPS-backed environment file containing Attic JWT and object-storage
+        credentials.
+      '';
+    };
+
+    storage = {
+      bucket = lib.mkOption {
+        type = lib.types.str;
+        description = "Hetzner Object Storage bucket name used by Attic.";
+      };
+
+      endpoint = lib.mkOption {
+        type = lib.types.str;
+        description = "S3-compatible HTTPS endpoint for Hetzner Object Storage.";
+      };
+
+      region = lib.mkOption {
+        type = lib.types.str;
+        description = "Region name for Hetzner Object Storage.";
+      };
+    };
+
+  };
+
+  config = lib.mkIf cfg.enable {
+    services.atticd = {
+      enable          = true;
+      environmentFile = cfg.environmentFile;
+      settings = {
+        listen = "${cfg.listenAddress}:${toString cfg.port}";
+        allowed-hosts = [ cfg.hostName ];
+        api-endpoint = "https://${cfg.hostName}/";
+        compression.type = "zstd";
+        storage = {
+          type     = "s3";
+          bucket   = cfg.storage.bucket;
+          endpoint = cfg.storage.endpoint;
+          region   = cfg.storage.region;
+        };
+      };
+    };
+  };
+}

nixos/module/hectic/service/element-rtc.nix

@@ -0,0 +1,135 @@
+{
+  inputs,
+  flake,
+  self,
+}:
+{
+  pkgs,
+  lib,
+  config,
+  ...
+}: let
+  legacyCfg = config.hectic.services.matrix;
+  hasClusterCfg = config.hectic ? generic && config.hectic.generic ? matrix-cluster;
+  clusterCfg = if hasClusterCfg then config.hectic.generic.matrix-cluster else null;
+  clusterSynapseEnabled =
+    if hasClusterCfg
+    then clusterCfg.enable
+      && (if clusterCfg.overrideEnableSynapse != null then clusterCfg.overrideEnableSynapse else clusterCfg.role == "primary")
+    else false;
+  enabled = legacyCfg.enable || clusterSynapseEnabled;
+  matrixDomain = if legacyCfg.enable then legacyCfg.matrixDomain else if hasClusterCfg then clusterCfg.matrixDomain else "";
+in {
+  config = lib.mkIf enabled (let
+    keyFile = "/run/livekit.key";
+  in {
+    services.livekit = {
+      enable = true;
+      openFirewall = true;
+      settings.room.auto_create = false;
+      inherit keyFile;
+    };
+
+    services.lk-jwt-service = {
+      enable = true;
+      livekitUrl = "wss://${matrixDomain}/livekit/sfu";
+      inherit keyFile;
+    };
+
+    systemd.services.livekit-key = {
+      before = [ "lk-jwt-service.service" "livekit.service" ];
+      wantedBy = [ "multi-user.target" ];
+      path = with pkgs; [ livekit coreutils gawk ];
+      script = ''
+        echo "Key missing, generating key"
+        echo "lk-jwt-service: $(livekit-server generate-keys | tail -1 | awk '{print $3}')" > "${keyFile}"
+      '';
+      serviceConfig.Type = "oneshot";
+      unitConfig.ConditionPathExists = "!${keyFile}";
+    };
+
+    systemd.services.lk-jwt-service.environment.LIVEKIT_FULL_ACCESS_HOMESERVERS =
+      matrixDomain;
+
+    services.nginx = {
+      enable = true;
+      virtualHosts.${matrixDomain} = {
+        forceSSL = true;
+        enableACME = true;
+
+        locations."=/.well-known/matrix/client" = {
+          extraConfig = ''
+            default_type application/json;
+            add_header Access-Control-Allow-Origin *;
+            add_header Access-Control-Allow-Methods "GET, POST, PUT, DELETE, OPTIONS";
+            add_header Access-Control-Allow-Headers "X-Requested-With, Content-Type, Authorization";
+          '';
+          return = ''200 '{
+            "m.homeserver": {
+              "base_url": "https://${matrixDomain}"
+            },
+            "m.identity_server": {
+              "base_url": "https://vector.im"
+            },
+            "org.matrix.msc3575.proxy": {
+              "url": "https://${matrixDomain}"
+            },
+            "org.matrix.msc4143.rtc_foci": [
+              {
+                "type": "livekit",
+                "livekit_service_url": "https://${matrixDomain}/livekit/jwt"
+              }
+            ]
+          }' '';
+        };
+
+        locations."= /livekit/jwt" = {
+          priority = 500;
+          proxyPass = "http://[::1]:${toString config.services.lk-jwt-service.port}/";
+        };
+
+        locations."^~ /livekit/jwt/" = {
+          priority = 400;
+          proxyPass = "http://[::1]:${toString config.services.lk-jwt-service.port}/";
+        };
+
+        locations."= /livekit/sfu" = {
+          priority = 500;
+          proxyPass = "http://[::1]:${toString config.services.livekit.settings.port}/";
+          proxyWebsockets = true;
+          extraConfig = ''
+            proxy_send_timeout 120;
+            proxy_read_timeout 120;
+            proxy_buffering off;
+            proxy_set_header Accept-Encoding gzip;
+            proxy_set_header Upgrade $http_upgrade;
+            proxy_set_header Connection "upgrade";
+          '';
+        };
+
+        locations."^~ /livekit/sfu/" = {
+          priority = 400;
+          proxyPass = "http://[::1]:${toString config.services.livekit.settings.port}/";
+          proxyWebsockets = true;
+          extraConfig = ''
+            proxy_send_timeout 120;
+            proxy_read_timeout 120;
+            proxy_buffering off;
+            proxy_set_header Accept-Encoding gzip;
+            proxy_set_header Upgrade $http_upgrade;
+            proxy_set_header Connection "upgrade";
+          '';
+        };
+      };
+    };
+
+    networking.firewall = {
+      enable = true;
+      allowedTCPPorts = [
+        8080
+        7880
+        7881
+      ];
+    };
+  });
+}

nixos/module/hectic/service/element.nix

@@ -0,0 +1,58 @@
+{
+  inputs,
+  flake,
+  self,
+}:
+{
+  pkgs,
+  lib,
+  config,
+  ...
+}: let
+  legacyCfg = config.hectic.services.matrix;
+  hasClusterCfg = config.hectic ? generic && config.hectic.generic ? matrix-cluster;
+  clusterCfg = if hasClusterCfg then config.hectic.generic.matrix-cluster else null;
+  clusterSynapseEnabled =
+    if hasClusterCfg
+    then clusterCfg.enable
+      && (if clusterCfg.overrideEnableSynapse != null then clusterCfg.overrideEnableSynapse else clusterCfg.role == "primary")
+    else false;
+  enabled = legacyCfg.enable || clusterSynapseEnabled;
+  matrixDomain = if legacyCfg.enable then legacyCfg.matrixDomain else if hasClusterCfg then clusterCfg.matrixDomain else "";
+  jitsiPreferredDomain =
+    if legacyCfg.enable && config.hectic.services.jitsi.enable
+    then config.hectic.services.jitsi.hostName
+    else if hasClusterCfg then clusterCfg.jitsi.preferredDomain else null;
+in {
+  config = lib.mkIf enabled {
+    services.nginx.virtualHosts."element.${matrixDomain}" = {
+      enableACME = true;
+      forceSSL = true;
+
+      locations."= /config.element.${matrixDomain}.json".return = "302 /config.json";
+
+      root = pkgs.hectic.element-web.override {
+        conf = {
+          default_server_config = {
+            "m.homeserver".base_url = "https://${matrixDomain}";
+            "m.homeserver".server_name = matrixDomain;
+            "m.identity_server".base_url = "https://vector.im";
+          };
+
+          room_directory.servers = [
+            matrixDomain
+          ];
+
+          hectic.videoMessages.enabled = true;
+
+          jitsi = lib.optionalAttrs (jitsiPreferredDomain != null) {
+            preferred_domain = jitsiPreferredDomain;
+          };
+
+          default_theme = "dark";
+          show_labs_settings = true;
+        };
+      };
+    };
+  };
+}

nixos/module/hectic/service/ente.nix

@@ -0,0 +1,224 @@
+{ ... }: {
+  lib,
+  config,
+  ...
+}: let
+  cfg = config.hectic.services.ente;
+
+  webHostNames = [
+    cfg.domains.accounts
+    cfg.domains.cast
+    cfg.domains.photos
+  ];
+in {
+  options.hectic.services.ente = {
+    enable = lib.mkEnableOption "Ente Photos self-hosted service";
+
+    apiDomain = lib.mkOption {
+      type = lib.types.str;
+      description = "Public hostname for the Ente Museum API.";
+    };
+
+    domains = {
+      accounts = lib.mkOption {
+        type = lib.types.str;
+        description = "Public hostname for the Ente accounts web app.";
+      };
+
+      cast = lib.mkOption {
+        type = lib.types.str;
+        description = "Public hostname for the Ente cast web app.";
+      };
+
+      albums = lib.mkOption {
+        type = lib.types.str;
+        description = "Public hostname for public Ente album links.";
+      };
+
+      photos = lib.mkOption {
+        type = lib.types.str;
+        description = "Public hostname for the Ente Photos web app.";
+      };
+    };
+
+    maxUploadSize = lib.mkOption {
+      type = lib.types.str;
+      default = "10G";
+      description = "Maximum request body accepted by nginx in front of Museum.";
+    };
+
+    disableRegistration = lib.mkOption {
+      type = lib.types.bool;
+      default = false;
+      description = "Whether Museum should reject new account registration.";
+    };
+
+    smtp = {
+      enable = lib.mkEnableOption "SMTP delivery for Ente verification emails";
+
+      host = lib.mkOption {
+        type = lib.types.str;
+        default = "127.0.0.1";
+        description = "SMTP host Museum uses to send verification emails.";
+      };
+
+      port = lib.mkOption {
+        type = lib.types.port;
+        default = 25;
+        description = "SMTP port Museum uses to send verification emails.";
+      };
+
+      email = lib.mkOption {
+        type = lib.types.str;
+        description = "From email address used by Museum.";
+      };
+
+      senderName = lib.mkOption {
+        type = lib.types.str;
+        default = "Ente Photos";
+        description = "Display name used for Ente verification emails.";
+      };
+
+      encryption = lib.mkOption {
+        type = lib.types.nullOr (lib.types.enum [ "tls" "ssl" ]);
+        default = null;
+        description = "Optional SMTP encryption mode. Leave null for local plaintext SMTP.";
+      };
+    };
+
+    storage = {
+      bucket = lib.mkOption {
+        type = lib.types.str;
+        description = "S3-compatible bucket used by Ente for photo object storage.";
+      };
+
+      endpoint = lib.mkOption {
+        type = lib.types.str;
+        description = "S3-compatible endpoint URL.";
+      };
+
+      region = lib.mkOption {
+        type = lib.types.str;
+        description = "S3-compatible region name.";
+      };
+
+      hotStorage = lib.mkOption {
+        type = lib.types.enum [
+          "b2-eu-cen"
+          "wasabi-eu-central-2-v3"
+          "scw-eu-fr-v3"
+        ];
+        default = "b2-eu-cen";
+        description = ''
+          Museum's primary hot-storage key. Upstream requires one of its
+          historical S3 storage identifiers even when the backing provider is a
+          generic S3-compatible service.
+        '';
+      };
+
+      usePathStyleUrls = lib.mkOption {
+        type = lib.types.bool;
+        default = false;
+        description = "Whether Museum should use path-style S3 URLs.";
+      };
+    };
+
+    secrets = {
+      encryptionKeyFile = lib.mkOption {
+        type = lib.types.path;
+        description = "File containing Museum key.encryption.";
+      };
+
+      hashKeyFile = lib.mkOption {
+        type = lib.types.path;
+        description = "File containing Museum key.hash.";
+      };
+
+      jwtSecretFile = lib.mkOption {
+        type = lib.types.path;
+        description = "File containing Museum jwt.secret.";
+      };
+
+      s3AccessKeyFile = lib.mkOption {
+        type = lib.types.path;
+        description = "File containing the S3 access key.";
+      };
+
+      s3SecretKeyFile = lib.mkOption {
+        type = lib.types.path;
+        description = "File containing the S3 secret key.";
+      };
+    };
+  };
+
+  config = lib.mkIf cfg.enable {
+    services.ente = {
+      api = {
+        enable        = true;
+        enableLocalDB = true;
+        domain        = cfg.apiDomain;
+
+        nginx.enable = true;
+
+        settings = {
+          key = {
+            encryption._secret = cfg.secrets.encryptionKeyFile;
+            hash._secret       = cfg.secrets.hashKeyFile;
+          };
+
+          jwt.secret._secret = cfg.secrets.jwtSecretFile;
+
+          s3 = {
+            hot_storage.primary = cfg.storage.hotStorage;
+            derived-storage     = cfg.storage.hotStorage;
+            are_local_buckets   = false;
+            use_path_style_urls = cfg.storage.usePathStyleUrls;
+
+            ${cfg.storage.hotStorage} = {
+              key._secret    = cfg.secrets.s3AccessKeyFile;
+              secret._secret = cfg.secrets.s3SecretKeyFile;
+              endpoint       = cfg.storage.endpoint;
+              region         = cfg.storage.region;
+              bucket         = cfg.storage.bucket;
+            };
+          };
+
+          internal.disable-registration = cfg.disableRegistration;
+
+          smtp = lib.mkIf cfg.smtp.enable ({
+            inherit (cfg.smtp) host port email;
+            sender-name = cfg.smtp.senderName;
+          } // lib.optionalAttrs (cfg.smtp.encryption != null) {
+            encryption = cfg.smtp.encryption;
+          });
+        };
+      };
+
+      web = {
+        enable = true;
+        domains = {
+          api = cfg.apiDomain;
+          inherit (cfg.domains) accounts cast albums photos;
+        };
+      };
+    };
+
+    services.nginx.virtualHosts =
+      (lib.genAttrs webHostNames (_: {
+        enableACME = true;
+        forceSSL   = true;
+      })) // {
+        ${cfg.apiDomain} = {
+          enableACME = true;
+          forceSSL   = true;
+          extraConfig = lib.mkForce ''
+            client_max_body_size ${cfg.maxUploadSize};
+          '';
+          locations."/".extraConfig = ''
+            proxy_read_timeout 600s;
+            proxy_send_timeout 600s;
+          '';
+        };
+      };
+  };
+}

nixos/module/hectic/service/jitsi.nix

@@ -0,0 +1,97 @@
+{
+  inputs,
+  flake,
+  self,
+}:
+{
+  pkgs,
+  lib,
+  config,
+  ...
+}: let
+  cfg = config.hectic.services.jitsi;
+in {
+  options = {
+    hectic.services.jitsi = {
+      enable = lib.mkEnableOption "Jitsi Meet video conferencing with Prosody XMPP backend";
+      hostName = lib.mkOption {
+        type        = lib.types.str;
+        description = ''
+          FQDN for the Jitsi Meet instance (e.g. "meet.example.org").
+          Prosody VirtualHosts, nginx, and ACME certs are derived from this.
+        '';
+      };
+      secureDomain = lib.mkOption {
+        type        = lib.types.bool;
+        default     = false;
+        description = ''
+          Require authentication to create rooms. Guests can still join
+          existing rooms anonymously.
+        '';
+      };
+      lockdown = lib.mkOption {
+        type        = lib.types.bool;
+        default     = false;
+        description = ''
+          Restrict Prosody to localhost only (no S2S federation, c2s
+          only on 127.0.0.1). Set to false when running alongside a
+          general-purpose XMPP server (hectic.services.xmpp).
+        '';
+      };
+      videobridgePasswordFile = lib.mkOption {
+        type        = lib.types.nullOr lib.types.path;
+        default     = null;
+        description = ''
+          Path to a file containing the Jitsi Videobridge XMPP password.
+          If null, a random password is auto-generated.
+        '';
+      };
+    };
+  };
+
+  config = lib.mkIf cfg.enable {
+    services.jitsi-meet = {
+      enable   = true;
+      hostName = cfg.hostName;
+
+      prosody = {
+        enable   = true;
+        lockdown = cfg.lockdown;
+      };
+
+      nginx.enable = true;
+      videobridge = {
+        enable = true;
+      } // lib.optionalAttrs (cfg.videobridgePasswordFile != null) {
+        passwordFile = cfg.videobridgePasswordFile;
+      };
+      jicofo.enable = true;
+
+      secureDomain = lib.mkIf cfg.secureDomain {
+        enable = true;
+      };
+    };
+
+    services.jitsi-videobridge.openFirewall = true;
+
+    services.nginx.virtualHosts.${cfg.hostName} = {
+      enableACME = true;
+      forceSSL   = true;
+    };
+
+    security.acme = {
+      acceptTerms = true;
+      defaults = {
+        email           = lib.mkDefault "hectic.yukkop.it@gmail.com";
+        enableDebugLogs = lib.mkDefault true;
+      };
+    };
+
+    networking.firewall = {
+      allowedTCPPorts = [
+        80 443   # HTTP/HTTPS (nginx + ACME)
+        5222     # XMPP c2s (if not locked down)
+      ];
+    };
+  };
+}

nixos/module/hectic/service/mailserver.nix

@@ -0,0 +1,75 @@
+{
+  inputs,
+  flake,
+  self,
+}:
+{
+  lib,
+  config,
+  ...
+}: let
+  cfg = config.services.mailserver;
+  transformLoginAccounts = domain: input:
+    builtins.listToAttrs (map (key: {
+      name  = key + "@" + domain;
+      value = input.${key};
+    }) (builtins.attrNames input));
+in {
+  options = {
+    services.mailserver.enable = lib.mkEnableOption "Mail server";
+
+    services.mailserver.domain = lib.mkOption {
+      type        = lib.types.str;
+      description = "The domain name of the mail server";
+    };
+
+    services.mailserver.loginAccounts = lib.mkOption {
+      type = lib.types.attrsOf (lib.types.submodule {
+        options = {
+          hashedPassword = lib.mkOption {
+            type    = lib.types.nullOr lib.types.str;
+            default = null;
+          };
+          hashedPasswordFile = lib.mkOption {
+            type        = lib.types.nullOr lib.types.str;
+            default     = null;
+            description = ''
+              Full path to a file containing the hashed password suitable
+              for use with `chpasswd -e`.
+            '';
+          };
+        };
+      });
+      default     = {};
+      description = "Login accounts for the mail server";
+    };
+  };
+
+  config = lib.mkIf cfg.enable {
+    mailserver = {
+      enable   = true;
+      fqdn     = "mail." + cfg.domain;
+      domains  = [ cfg.domain ];
+
+      loginAccounts = transformLoginAccounts cfg.domain cfg.loginAccounts;
+
+      certificateScheme = "acme-nginx";
+    };
+
+    services.postfix.settings.main = {
+      # NOTE(yukkop): avoid Gmail rejection due to missing IPv6 PTR records.
+      inet_protocols = lib.mkDefault "ipv4";
+
+      # NOTE(yukkop): nixos-mailserver enables DANE by default. Some large MXes
+      # currently fail certificate verification under this policy, which leaves
+      # otherwise valid transactional mail deferred in the queue. Keep STARTTLS
+      # opportunistic for outbound delivery rather than blocking mail entirely.
+      smtp_tls_security_level = lib.mkForce "may";
+      smtp_dns_support_level  = lib.mkForce "enabled";
+      smtp_tls_policy_maps    = lib.mkForce "";
+    };
+
+    security.acme.acceptTerms       = true;
+    security.acme.defaults.email    = "security@" + cfg.domain;
+  };
+}

nixos/module/hectic/service/matrix.nix

@@ -0,0 +1,508 @@
+{
+  inputs,
+  flake,
+  self,
+}:
+{
+  pkgs,
+  lib,
+  config,
+  ...
+}: let
+  cfg = config.hectic.services.matrix;
+  s3Cfg = cfg.objectStorage.s3;
+
+  matrixUsers = builtins.attrNames cfg.users;
+
+  s3Plugin = pkgs.matrix-synapse-plugins.matrix-synapse-s3-storage-provider;
+  s3ConfigDir = "/run/matrix-synapse";
+  s3ConfigFile = "${s3ConfigDir}/s3-media-storage.yaml";
+
+  mkUserRegistration = name: let
+    user = cfg.users.${name};
+    adminFlag = if user.admin then "--admin" else "--no-admin";
+  in ''
+    if [ ! -r "${user.passwordFile}" ]; then
+      printf 'Missing Matrix password file for %s: %s\n' '${name}' '${user.passwordFile}' >&2
+      exit 1
+    fi
+
+    ${pkgs.matrix-synapse}/bin/register_new_matrix_user \
+      -u '${name}' \
+      -p "$(tr -d '\n' < "${user.passwordFile}")" \
+      -k "$REGISTRATION_SHARED_SECRET" \
+      ${adminFlag} \
+      http://127.0.0.1:8008 || true
+  '';
+
+  mkS3Config = ''
+    if [ ! -r "${s3Cfg.credentialsFile}" ]; then
+      printf 'Missing Matrix object storage credentials file: %s\n' '${s3Cfg.credentialsFile}' >&2
+      exit 1
+    fi
+
+    . "${s3Cfg.credentialsFile}"
+
+    if [ -z "$ACCESS_KEY_ID" ] || [ -z "$SECRET_ACCESS_KEY" ]; then
+      printf 'ACCESS_KEY_ID or SECRET_ACCESS_KEY missing in %s\n' '${s3Cfg.credentialsFile}' >&2
+      exit 1
+    fi
+
+    mkdir -p "${s3ConfigDir}"
+
+    cat > "${s3ConfigFile}" <<EOF
+    media_storage_providers:
+      - module: s3_storage_provider.S3StorageProviderBackend
+        store_local: ${lib.boolToString s3Cfg.storeLocal}
+        store_remote: ${lib.boolToString s3Cfg.storeRemote}
+        store_synchronous: ${lib.boolToString s3Cfg.storeSynchronous}
+        config:
+          bucket: ${s3Cfg.bucket}
+          endpoint_url: ${s3Cfg.endpointUrl}
+          region_name: ${s3Cfg.regionName}
+          prefix: "${s3Cfg.prefix}"
+          storage_class: "${s3Cfg.storageClass}"
+          threadpool_size: ${toString s3Cfg.threadpoolSize}
+          access_key_id: $ACCESS_KEY_ID
+          secret_access_key: $SECRET_ACCESS_KEY
+    EOF
+
+    chown matrix-synapse:matrix-synapse "${s3ConfigFile}"
+    chmod 0400 "${s3ConfigFile}"
+  '';
+
+  mkS3SyncScript = ''
+    ${s3Plugin}/bin/s3_media_upload write
+    ${s3Plugin}/bin/s3_media_upload upload "${s3Cfg.mediaStorePath}" "${s3Cfg.bucket}" \
+      --endpoint-url "${s3Cfg.endpointUrl}" \
+      --storage-class "${s3Cfg.storageClass}" \
+      --prefix "${s3Cfg.prefix}" \
+      ${lib.optionalString s3Cfg.sync.deleteLocalAfterUpload "--delete"}
+    cat > /tmp/synapse-merge-config.py << 'PYEOF'
+import yaml
+with open("${config.services.matrix-synapse.configFile}") as f:
+    config = yaml.safe_load(f)
+with open("${cfg.secretsFile}") as f:
+    secrets = yaml.safe_load(f)
+config.update(secrets)
+config.setdefault("database", {}).setdefault("args", {})
+config["database"]["args"].setdefault("password", "")
+config["database"]["args"].setdefault("host", "/run/postgresql")
+config["database"]["args"].setdefault("port", 5432)
+with open("/tmp/synapse-combined-config.yaml", "w") as f:
+    yaml.dump(config, f, default_flow_style=False)
+PYEOF
+    ${pkgs.python3.withPackages (ps: [ps.pyyaml])}/bin/python3 /tmp/synapse-merge-config.py
+    ${s3Plugin}/bin/s3_media_upload update-db --homeserver-config-path /tmp/synapse-combined-config.yaml 0s
+    rm -f /tmp/synapse-combined-config.yaml
+    ${s3Plugin}/bin/s3_media_upload check-deleted "${s3Cfg.mediaStorePath}"
+  '';
+in {
+  options = {
+    hectic.services.matrix = {
+      enable = lib.mkEnableOption "Matrix Synapse homeserver with PostgreSQL and nginx";
+
+      secretsFile = lib.mkOption {
+        type = lib.types.path;
+        description = ''
+          path to env file with matrix secrets
+
+          content:
+          registration_shared_secret:
+          macroon_secret_key
+          form_secret
+        '';
+      };
+
+      postgresql = {
+        port = lib.mkOption {
+          type = lib.types.port;
+          default = 5432;
+          description = ''
+            postgres port
+          '';
+        };
+
+        initialEnvFile = lib.mkOption {
+          type = lib.types.path;
+          description = ''
+            path to env file with postgresql initial secrets
+
+            content:
+            POSTGRESQL_PASSWORD=
+          '';
+        };
+      };
+
+      matrixDomain = lib.mkOption {
+        type = lib.types.str;
+        description = ''
+          domain to matrix
+        '';
+      };
+
+      maxUploadSize = lib.mkOption {
+        type = lib.types.str;
+        default = "2G";
+        description = ''
+          Maximum file upload size accepted by Synapse and nginx.
+        '';
+      };
+
+      enableRegistration = lib.mkOption {
+        type = lib.types.bool;
+        default = false;
+        description = ''
+          Allow open user registration on the homeserver.
+        '';
+      };
+
+      objectStorage.s3 = {
+        enable = lib.mkEnableOption "S3-compatible object storage for Matrix media";
+
+        bucket = lib.mkOption {
+          type = lib.types.str;
+          description = ''
+            Bucket name used for Matrix media objects.
+          '';
+        };
+
+        regionName = lib.mkOption {
+          type = lib.types.str;
+          description = ''
+            Region name passed to the Synapse S3 storage provider.
+          '';
+        };
+
+        endpointUrl = lib.mkOption {
+          type = lib.types.str;
+          description = ''
+            S3-compatible endpoint URL.
+          '';
+        };
+
+        credentialsFile = lib.mkOption {
+          type = lib.types.path;
+          description = ''
+            Path to an env-style file containing:
+              ACCESS_KEY_ID=
+              SECRET_ACCESS_KEY=
+          '';
+        };
+
+        mediaStorePath = lib.mkOption {
+          type = lib.types.str;
+          default = "/var/lib/matrix-synapse/media_store";
+          description = ''
+            Local Synapse media store path used before upload to object storage.
+          '';
+        };
+
+        prefix = lib.mkOption {
+          type = lib.types.str;
+          default = "";
+          description = ''
+            Optional object key prefix inside the bucket.
+          '';
+        };
+
+        storageClass = lib.mkOption {
+          type = lib.types.str;
+          default = "STANDARD";
+          description = ''
+            Storage class passed to the upload tool.
+          '';
+        };
+
+        threadpoolSize = lib.mkOption {
+          type = lib.types.int;
+          default = 40;
+          description = ''
+            Worker pool size for the Synapse S3 storage provider.
+          '';
+        };
+
+        storeLocal = lib.mkOption {
+          type = lib.types.bool;
+          default = true;
+          description = ''
+            Mirror local uploads to object storage.
+          '';
+        };
+
+        storeRemote = lib.mkOption {
+          type = lib.types.bool;
+          default = true;
+          description = ''
+            Mirror remotely-fetched media to object storage.
+          '';
+        };
+
+        storeSynchronous = lib.mkOption {
+          type = lib.types.bool;
+          default = true;
+          description = ''
+            Wait for object storage upload before completing the client request.
+          '';
+        };
+
+        sync = {
+          enable = lib.mkOption {
+            type = lib.types.bool;
+            default = true;
+            description = ''
+              Periodically migrate older local media to object storage.
+            '';
+          };
+
+          olderThan = lib.mkOption {
+            type = lib.types.str;
+            default = "1d";
+            description = ''
+              Age threshold passed to `s3_media_upload update`.
+            '';
+          };
+
+          onCalendar = lib.mkOption {
+            type = lib.types.str;
+            default = "hourly";
+            description = ''
+              systemd timer schedule for media sync.
+            '';
+          };
+
+          deleteLocalAfterUpload = lib.mkOption {
+            type = lib.types.bool;
+            default = true;
+            description = ''
+              Remove local media after successful object storage upload.
+            '';
+          };
+        };
+      };
+
+      users = lib.mkOption {
+        type = lib.types.attrsOf (lib.types.submodule {
+          options = {
+            passwordFile = lib.mkOption {
+              type = lib.types.str;
+              description = ''
+                Full path to a file containing the Matrix user's password.
+              '';
+            };
+
+            admin = lib.mkOption {
+              type = lib.types.bool;
+              default = false;
+              description = ''
+                Whether to create the Matrix user as an admin.
+              '';
+            };
+          };
+        });
+        default = {};
+        description = ''
+          Declarative Matrix users to provision after Synapse starts.
+        '';
+      };
+    };
+  };
+
+  config = lib.mkMerge [
+    (lib.mkIf cfg.enable {
+      services.matrix-synapse = {
+        enable = true;
+        plugins = lib.optional s3Cfg.enable s3Plugin;
+        extraConfigFiles = [
+          cfg.secretsFile
+        ] ++ lib.optional s3Cfg.enable s3ConfigFile;
+
+        settings = {
+          server_name = cfg.matrixDomain;
+          public_baseurl = "https://${cfg.matrixDomain}";
+          max_upload_size = cfg.maxUploadSize;
+          media_store_path = lib.mkIf s3Cfg.enable s3Cfg.mediaStorePath;
+
+          experimental_features = {
+            msc3266_enabled = true;
+            msc4140_enabled = true;
+            msc4143_enabled = true;
+            msc4222_enabled = true;
+          };
+
+          matrix_rtc = {
+            transports = [
+              {
+                type = "livekit";
+                livekit_service_url = "https://${cfg.matrixDomain}/livekit/jwt";
+              }
+            ];
+          };
+
+          listeners = [
+            {
+              port = 8008;
+              bind_addresses = [ "0.0.0.0" ];
+              type = "http";
+              tls = false;
+              resources = [
+                {
+                  names = [
+                    "client"
+                    # Ability speak between different matrix servers and get
+                    # global id, requires .well-known
+                    "federation"
+                    "openid"
+                  ];
+                  compress = false;
+                }
+              ];
+            }
+          ];
+
+          enable_registration = cfg.enableRegistration;
+          enable_registration_without_verification = cfg.enableRegistration;
+        };
+      };
+
+      environment.systemPackages = [
+        pkgs.matrix-synapse
+      ];
+
+      services.postgresql = {
+        enable = true;
+        package = pkgs.postgresql_17;
+
+        initdbArgs = [
+          "--locale=C"
+          "--encoding=UTF8"
+        ];
+
+        enableTCPIP = true;
+        settings.port = cfg.postgresql.port;
+        authentication = builtins.concatStringsSep "\n" [
+          "local all         all           trust"
+          "host  sameuser    all           127.0.0.1/32 scram-sha-256"
+          "host  sameuser    all           ::1/128 scram-sha-256"
+          "host  all         all           ::1/128 scram-sha-256"
+          "host  all         all           0.0.0.0/0 scram-sha-256"
+
+          "host  replication postgres      127.0.0.1/32   scram-sha-256"
+          "host  replication postgres      ::1/128        scram-sha-256"
+        ];
+
+        settings = {
+          wal_level = "replica";
+          max_wal_senders = 10;
+        };
+
+        ensureUsers = [
+          {
+            name = "matrix-synapse";
+            ensureClauses.login = true;
+            ensureDBOwnership = true;
+          }
+        ];
+
+        ensureDatabases = [
+          "matrix-synapse"
+        ];
+
+        initialScript = pkgs.writeText "init-sql-script" ''
+          -- setup password from env/sops
+          DO $$#!${pkgs.dash}/bin/dash
+            set -e
+            . ${cfg.postgresql.initialEnvFile}
+            psql -Atc "ALTER USER postgres WITH PASSWORD '$POSTGRESQL_PASSWORD'";
+          $$ LANGUAGE plsh;
+
+          CREATE ROLE myuser LOGIN PASSWORD 'matrix-synapse';
+        '';
+      };
+
+      services.nginx = {
+        enable = true;
+        virtualHosts.${cfg.matrixDomain} = {
+          forceSSL = true;
+          enableACME = true;
+          locations."/" = {
+            proxyPass = "http://127.0.0.1:8008";
+            extraConfig = ''
+              client_max_body_size ${cfg.maxUploadSize};
+            '';
+          };
+          locations."=/.well-known/matrix/server" = {
+            extraConfig = ''
+              default_type application/json;
+              add_header Access-Control-Allow-Origin *;
+              add_header Access-Control-Allow-Methods "GET, POST, PUT, DELETE, OPTIONS";
+              add_header Access-Control-Allow-Headers "X-Requested-With, Content-Type, Authorization";
+            '';
+            return = "200 '{\"m.server\": \"${cfg.matrixDomain}:443\"}'";
+          };
+        };
+      };
+
+      security.acme = {
+        acceptTerms = true;
+        defaults = {
+          email = "hectic.yukkop.it@gmail.com";
+          enableDebugLogs = true;
+        };
+      };
+
+      systemd.services.matrix-synapse-users = lib.mkIf (matrixUsers != []) {
+        description = "Provision Matrix Synapse users";
+        wantedBy = [ "multi-user.target" ];
+        after = [ config.services.matrix-synapse.serviceUnit ];
+        requires = [ config.services.matrix-synapse.serviceUnit ];
+        path = with pkgs; [ curl coreutils gawk ];
+        serviceConfig = {
+          Type = "oneshot";
+          User = "matrix-synapse";
+        };
+        script = ''
+          until curl -sf http://127.0.0.1:8008/_matrix/client/versions >/dev/null; do
+            sleep 2
+          done
+
+          REGISTRATION_SHARED_SECRET="$(awk -F': *' '$1 == "registration_shared_secret" { print $2; exit }' "${cfg.secretsFile}")"
+
+          if [ -z "$REGISTRATION_SHARED_SECRET" ]; then
+            printf 'registration_shared_secret not found in %s\n' '${cfg.secretsFile}' >&2
+            exit 1
+          fi
+
+${builtins.concatStringsSep "\n" (map mkUserRegistration matrixUsers)}
+        '';
+      };
+    })
+
+    (lib.mkIf (cfg.enable && s3Cfg.enable) {
+      systemd.services.matrix-synapse-s3-config = {
+        description = "Generate Synapse S3 media storage config";
+        before = [ config.services.matrix-synapse.serviceUnit ];
+        requiredBy = [ config.services.matrix-synapse.serviceUnit ];
+        serviceConfig.Type = "oneshot";
+        script = mkS3Config;
+      };
+
+      systemd.services.matrix-synapse-s3-media-sync = lib.mkIf s3Cfg.sync.enable {
+        description = "Sync Matrix media to S3-compatible object storage";
+        after = [ config.services.matrix-synapse.serviceUnit ];
+        wants = [ config.services.matrix-synapse.serviceUnit ];
+        serviceConfig = {
+          Type = "oneshot";
+          User = "matrix-synapse";
+          WorkingDirectory = "/var/lib/matrix-synapse";
+        };
+        script = mkS3SyncScript;
+      };
+
+      systemd.timers.matrix-synapse-s3-media-sync = lib.mkIf s3Cfg.sync.enable {
+        wantedBy = [ "timers.target" ];
+        timerConfig.OnCalendar = s3Cfg.sync.onCalendar;
+      };
+    })
+  ];
+}

nixos/module/hectic/service/media-browser.nix

@@ -0,0 +1,99 @@
+{
+  inputs,
+  flake,
+  self,
+}: {
+  pkgs,
+  lib,
+  config,
+  ...
+}: let
+  cfg = config.hectic.services.media-browser;
+
+  mediaBrowserApp = pkgs.hectic.media-browser;
+in {
+  options.hectic.services.media-browser = {
+    enable = lib.mkEnableOption "Matrix media browser web app";
+
+    port = lib.mkOption {
+      type = lib.types.port;
+      default = 3000;
+      description = "Port to bind the media browser web server.";
+    };
+
+    mediaStorePath = lib.mkOption {
+      type = lib.types.str;
+      default = "/var/lib/matrix-synapse/media_store";
+      description = "Path to Synapse media store.";
+    };
+
+    s3CredentialsFile = lib.mkOption {
+      type = lib.types.path;
+      description = "Path to S3 credentials file (ACCESS_KEY_ID=..., SECRET_ACCESS_KEY=...).";
+    };
+
+    s3Bucket = lib.mkOption {
+      type = lib.types.str;
+      description = "S3 bucket name.";
+    };
+
+    s3Endpoint = lib.mkOption {
+      type = lib.types.str;
+      description = "S3 endpoint URL.";
+    };
+
+    s3Region = lib.mkOption {
+      type = lib.types.str;
+      default = "hel1";
+      description = "S3 region name.";
+    };
+
+    s3Prefix = lib.mkOption {
+      type = lib.types.str;
+      default = "";
+      description = "S3 object key prefix.";
+    };
+
+    dbName = lib.mkOption {
+      type = lib.types.str;
+      default = "matrix-synapse";
+      description = "PostgreSQL database name.";
+    };
+
+    dbUser = lib.mkOption {
+      type = lib.types.str;
+      default = "matrix-synapse";
+      description = "PostgreSQL database user.";
+    };
+  };
+
+  config = lib.mkIf cfg.enable {
+    systemd.services.media-browser = {
+      description = "Matrix Media Browser";
+      after = [ "network.target" "postgresql.target" ];
+      wantedBy = [ "multi-user.target" ];
+      serviceConfig = {
+        Type = "simple";
+        User = "matrix-synapse";
+        Group = "matrix-synapse";
+        ExecStart = "${mediaBrowserApp}/bin/media-browser-wrapped";
+        Restart = "on-failure";
+        RestartSec = 5;
+      };
+      environment = {
+        FLASK_ENV = "production";
+        PORT = toString cfg.port;
+        MEDIA_STORE_PATH = cfg.mediaStorePath;
+        S3_BUCKET = cfg.s3Bucket;
+        S3_ENDPOINT = cfg.s3Endpoint;
+        S3_REGION = cfg.s3Region;
+        S3_PREFIX = cfg.s3Prefix;
+        DB_NAME = cfg.dbName;
+        DB_USER = cfg.dbUser;
+        DB_HOST = "/run/postgresql";
+        DB_PORT = "5432";
+      };
+      serviceConfig.EnvironmentFile = cfg.s3CredentialsFile;
+    };
+  };
+}

nixos/module/hectic/service/sentinèlla.nix

@@ -0,0 +1,187 @@
+{ 
+  inputs,
+  flake,
+  self,
+}:
+{ 
+  pkgs,
+  lib,
+  config,
+  ...
+}: let
+  system = pkgs.stdenv.hostPlatform.system;
+  cfg    = config.hectic.services."sentinèlla";
+
+  probePort = 5988;
+  peersSrv  = "_sentinella._tcp.hectic-lab.com";
+in {
+  options = {
+    hectic.services."sentinèlla" = {
+      probe = {
+        enable = lib.mkEnableOption "sentinèlla probe — HTTP server exposing this node's health";
+        urls = lib.mkOption {
+          type    = with lib.types; listOf str;
+          default = [];
+          description = "URLs the probe health-checks on GET /status.";
+        };
+        volumes = lib.mkOption {
+          type    = with lib.types; listOf str;
+          default = [];
+          description = "Mount points reported on GET /disk. Empty means all volumes.";
+        };
+        authFile = lib.mkOption {
+          type    = with lib.types; nullOr path;
+          default = null;
+          example = "config.sops.secrets.\"sentinella-probe-auth\".path";
+          description = "Path to a file with lines of the form user:pass for Basic Auth.";
+        };
+        environmentFile = lib.mkOption {
+          type    = with lib.types; nullOr path;
+          default = null;
+          description = ''
+            Optional environment file for secrets. Supported variables:
+              PORT=
+              URLS=
+              VOLUMES=
+              AUTH_FILE=
+          '';
+        };
+      };
+
+      watcher = {
+        enable = lib.mkEnableOption "sentinèlla watcher — polls peers discovered via DNS and sends Telegram alerts";
+        self = lib.mkOption {
+          type    = with lib.types; nullOr str;
+          default = null;
+          example = "1.2.3.4";
+          description = ''
+            Override the auto-detected local IP. When null (default) the watcher
+            uses hostname -I to find all local IPs and excludes them from the
+            peer list automatically. Set this only if the node is behind NAT or
+            has a floating IP that hostname -I does not report correctly.
+          '';
+        };
+        peersScheme = lib.mkOption {
+          type    = lib.types.str;
+          default = "http";
+          description = "URL scheme used when connecting to peers (http or https).";
+        };
+        pollingIntervalSec = lib.mkOption {
+          type    = lib.types.int;
+          default = 3;
+          description = "Seconds between polling rounds.";
+        };
+        tgToken = lib.mkOption {
+          type    = with lib.types; nullOr str;
+          default = null;
+          description = "Telegram bot token. Prefer environmentFile for secrets.";
+        };
+        tgChatId = lib.mkOption {
+          type    = with lib.types; nullOr str;
+          default = null;
+          description = "Telegram chat ID. Prefer environmentFile for secrets.";
+        };
+        environmentFile = lib.mkOption {
+          type    = with lib.types; nullOr path;
+          default = config.sops.secrets."sentinèlla/watcher/environment".path;
+          defaultText = lib.literalExpression
+            "config.sops.secrets.\"sentinèlla/watcher/environment\".path";
+          example = "config.sops.secrets.\"sentinella-watcher-env\".path";
+          description = ''
+            Environment file for secrets. Defaults to the auto-declared SOPS
+            secret sentinèlla/watcher/environment (resolved from
+            sus/sentinella-default.yaml in the flake). Override the sopsFile
+            via sops.secrets."sentinèlla/watcher/environment".sopsFile if you
+            need a host-specific file instead.
+
+            Supported variables:
+              TG_TOKEN=
+              TG_CHAT_ID=
+              PEERS_TOKEN=   # Basic Auth token sent to all peers
+              SELF=
+              PEERS_SRV=
+          '';
+        };
+      };
+    };
+  };
+
+  config = lib.mkMerge [
+    (lib.mkIf cfg.probe.enable {
+      networking.firewall = {
+        enable = true;
+        allowedTCPPorts = [
+          probePort     
+        ];
+      };
+
+      systemd.services."sentinella-probe" = {
+        description = "sentinèlla probe — node health HTTP server";
+        after       = [ "network.target" ];
+        wantedBy    = [ "multi-user.target" ];
+        serviceConfig = lib.mkMerge [
+          {
+            Type            = "simple";
+            ExecStart       = "${self.packages.${system}."sentinèlla"}/bin/probe";
+            Restart         = "always";
+            RestartSec      = "5s";
+            TimeoutStopSec  = "30s";
+            KillSignal      = "SIGTERM";
+            KillMode        = "mixed";
+            RemainAfterExit = false;
+            StandardOutput  = "journal";
+            StandardError   = "journal";
+            Environment = lib.filter (s: s != "") [
+              "PORT=${builtins.toString probePort}"
+              (lib.optionalString (cfg.probe.urls    != []) "URLS=${lib.concatStringsSep " " cfg.probe.urls}")
+              (lib.optionalString (cfg.probe.volumes != []) "VOLUMES=${lib.concatStringsSep " " cfg.probe.volumes}")
+              (lib.optionalString (cfg.probe.authFile != null) "AUTH_FILE=${cfg.probe.authFile}")
+            ];
+          }
+          (lib.mkIf (cfg.probe.environmentFile != null) {
+            EnvironmentFile = cfg.probe.environmentFile;
+          })
+        ];
+      };
+    })
+
+    (lib.mkIf cfg.watcher.enable {
+      sops.secrets."sentinèlla/watcher/environment" = lib.mkDefault {
+        sopsFile = flake + "/sus/sentinella-default.yaml";
+      };
+
+      systemd.services."sentinella-watcher" = {
+        description = "sentinèlla watcher — p2p peer monitor";
+        after       = [ "network.target" ];
+        wantedBy    = [ "multi-user.target" ];
+        serviceConfig = lib.mkMerge [
+          {
+            Type            = "simple";
+            ExecStart       = "${self.packages.${system}."sentinèlla"}/bin/watcher";
+            Restart         = "always";
+            RestartSec      = "5s";
+            TimeoutStopSec  = "30s";
+            KillSignal      = "SIGTERM";
+            KillMode        = "mixed";
+            RemainAfterExit = false;
+            StandardOutput  = "journal";
+            StandardError   = "journal";
+            StateDirectory  = "sentinella";
+            Environment = lib.filter (s: s != "") [
+              "PEERS_SRV=${peersSrv}"
+              (lib.optionalString (cfg.watcher.self != null) "SELF=${cfg.watcher.self}")
+              "PEERS_SCHEME=${cfg.watcher.peersScheme}"
+              "POLLING_INTERVAL_SEC=${builtins.toString cfg.watcher.pollingIntervalSec}"
+              "STATE_DIR=/var/lib/sentinella"
+              (lib.optionalString (cfg.watcher.tgToken  != null) "TG_TOKEN=${cfg.watcher.tgToken}")
+              (lib.optionalString (cfg.watcher.tgChatId != null) "TG_CHAT_ID=${cfg.watcher.tgChatId}")
+            ];
+          }
+          (lib.mkIf (cfg.watcher.environmentFile != null) {
+            EnvironmentFile = cfg.watcher.environmentFile;
+          })
+        ];
+      };
+    })
+  ];
+}