feat: lab: generic functions for mail

The hectic logger opens fd 3 as a dup of stderr. Child processes inherit
this fd, and daemonized PostgreSQL/PostgREST keeping it open prevents the
terminal from returning to the prompt after the spawning script exits.

- Add with_closed_fds helper that runs commands in a subshell with fds
  3-9 redirected to /dev/null
- Inline the helper into both database and postgres-init builds
- Wrap pg_ctl start and postgrest with the helper

nixos/module/generic/matrix-cluster-users.nix

@@ -18,26 +18,30 @@
   ];
 
   adminNames = [ "yukkop" ];
-in {
-  hectic.generic.matrix-cluster.users = builtins.listToAttrs (
-    map (name: {
-      inherit name;
-      value = {
-        passwordFile = config.sops.secrets."matrix/users/${name}/password".path;
-      } // lib.optionalAttrs (builtins.elem name adminNames) {
-        admin = true;
-      };
-    }) userNames
-  );
 
-  sops.secrets = builtins.listToAttrs (
-    map (name: {
-      name = "matrix/users/${name}/password";
-      value = {
-        key      = "matrix/users/${name}/password";
-        owner    = "matrix-synapse";
-        sopsFile = "${flake}/sus/matrix-cluster.yaml";
-      };
-    }) userNames
-  );
+  cfg = config.hectic.generic.matrix-cluster;
+in {
+  config = lib.mkIf cfg.enable {
+    hectic.generic.matrix-cluster.users = builtins.listToAttrs (
+      map (name: {
+        inherit name;
+        value = {
+          passwordFile = config.sops.secrets."matrix/users/${name}/password".path;
+        } // lib.optionalAttrs (builtins.elem name adminNames) {
+          admin = true;
+        };
+      }) userNames
+    );
+
+    sops.secrets = builtins.listToAttrs (
+      map (name: {
+        name = "matrix/users/${name}/password";
+        value = {
+          key      = "matrix/users/${name}/password";
+          owner    = "matrix-synapse";
+          sopsFile = "${flake}/sus/matrix-cluster.yaml";
+        };
+      }) userNames
+    );
+  }
 }

nixos/module/hectic/hardware/hetzner-cloud.nix

@@ -10,9 +10,21 @@
   ...
 }: let
   cfg = config.hectic.hardware.hetzner-cloud;
+  isNewer = cfg.generation == "newer";
 in {
   options.hectic.hardware.hetzner-cloud = {
     enable = lib.mkEnableOption "Enable hetzner-cloud hardware configurations";
+    generation = lib.mkOption {
+      type = lib.types.enum [ "classic" "newer" ];
+      default = "classic";
+      description = ''
+        Hetzner server generation profile.
+
+        `classic` keeps the historical `/dev/sda` assumption.
+        `newer` is for ccx/NVMe-era servers and defaults the disk device to
+        `/dev/nvme0n1`.
+      '';
+    };
     #bootParUuid = lib.mkOption {
     #  type = with lib.types; nullOr oneOf [
     #    (lib.types.strMatching "^[0-9A-Fa-f]{4}-[0-9A-Fa-f]{4}$")
@@ -41,22 +53,23 @@ in {
     };
     device = lib.mkOption {
       type = lib.types.str;
-      default = "/dev/sda";
+      default = if isNewer then "/dev/nvme0n1" else "/dev/sda";
       example = "/dev/disk/by-uuid/f184a16b-6eca-41cb-b48a-ff37cdce1d79";
       description = ''
         boot device uuid 
         if it is null then will use "/dev/sda" 
         /dev/sda - default hetzner cloud device
+        /dev/nvme0n1 - default for newer Hetzner generations
         !! But can changes on reboot if server have volumes
         !! So use IDs
       '';
     };
     networkMatchConfigName = lib.mkOption {
-      type = lib.types.strMatching "^(enp1s0|ens3)$";
+      type = lib.types.strMatching "^(enp1s0|ens3|eth0)$";
       example = "enp1s0";
       description = ''
         type of network conection, 
-        on older hetzner servers may be `ens3`
+        on older hetzner servers may be `ens3` or else
         on newer probably `enp1s0`
 
         you can use `networkctl list` on server to know it

nixos/module/hectic/hardware/zomro.nix

@@ -9,10 +9,10 @@
   config,
   ...
 }: let
-  cfg = config.hectic.hardware.zombro;
+  cfg = config.hectic.hardware.zomro;
 in {
-  options.hectic.hardware.zombro = {
-    enable = lib.mkEnableOption "Enable zombro hardware configurations";
+  options.hectic.hardware.zomro = {
+    enable = lib.mkEnableOption "Enable zomro hardware configurations";
     device = lib.mkOption {
       type = lib.types.str;
       default = "/dev/vda";
@@ -20,7 +20,7 @@ in {
       description = ''
         boot device uuid 
         if it is null then will use "/dev/vda" 
-        /dev/vda - default zombro device
+        /dev/vda - default zomro device
         !! But can changes on reboot if server have volumes
         !! So use IDs
       '';

nixos/module/hectic/service/element-rtc.nix

@@ -10,12 +10,15 @@
   ...
 }: let
   legacyCfg = config.hectic.services.matrix;
-  clusterCfg = config.hectic.generic.matrix-cluster;
+  hasClusterCfg = config.hectic ? generic && config.hectic.generic ? matrix-cluster;
+  clusterCfg = if hasClusterCfg then config.hectic.generic.matrix-cluster else null;
   clusterSynapseEnabled =
-    clusterCfg.enable
-    && (if clusterCfg.overrideEnableSynapse != null then clusterCfg.overrideEnableSynapse else clusterCfg.role == "primary");
+    if hasClusterCfg
+    then clusterCfg.enable
+      && (if clusterCfg.overrideEnableSynapse != null then clusterCfg.overrideEnableSynapse else clusterCfg.role == "primary")
+    else false;
   enabled = legacyCfg.enable || clusterSynapseEnabled;
-  matrixDomain = if legacyCfg.enable then legacyCfg.matrixDomain else clusterCfg.matrixDomain;
+  matrixDomain = if legacyCfg.enable then legacyCfg.matrixDomain else if hasClusterCfg then clusterCfg.matrixDomain else "";
 in {
   config = lib.mkIf enabled (let
     keyFile = "/run/livekit.key";

nixos/module/hectic/service/element.nix

@@ -10,16 +10,19 @@
   ...
 }: let
   legacyCfg = config.hectic.services.matrix;
-  clusterCfg = config.hectic.generic.matrix-cluster;
+  hasClusterCfg = config.hectic ? generic && config.hectic.generic ? matrix-cluster;
+  clusterCfg = if hasClusterCfg then config.hectic.generic.matrix-cluster else null;
   clusterSynapseEnabled =
-    clusterCfg.enable
-    && (if clusterCfg.overrideEnableSynapse != null then clusterCfg.overrideEnableSynapse else clusterCfg.role == "primary");
+    if hasClusterCfg
+    then clusterCfg.enable
+      && (if clusterCfg.overrideEnableSynapse != null then clusterCfg.overrideEnableSynapse else clusterCfg.role == "primary")
+    else false;
   enabled = legacyCfg.enable || clusterSynapseEnabled;
-  matrixDomain = if legacyCfg.enable then legacyCfg.matrixDomain else clusterCfg.matrixDomain;
+  matrixDomain = if legacyCfg.enable then legacyCfg.matrixDomain else if hasClusterCfg then clusterCfg.matrixDomain else "";
   jitsiPreferredDomain =
     if legacyCfg.enable && config.hectic.services.jitsi.enable
     then config.hectic.services.jitsi.hostName
-    else clusterCfg.jitsi.preferredDomain;
+    else if hasClusterCfg then clusterCfg.jitsi.preferredDomain else null;
 in {
   config = lib.mkIf enabled {
     services.nginx.virtualHosts."element.${matrixDomain}" = {

nixos/system/hectic-lab/hectic-lab.nix

@@ -16,6 +16,26 @@ with lib;
 let
   domain = "hectic-lab.com";
   matrixDomain = "accord.tube";
+  mailUserNames = [
+    "security"
+    "founders"
+    "lvgkcfjl"
+    "yukkop"
+    "daniil-perlyk"
+    "iana-perlyk"
+    "snuff"
+    "antoshka"
+  ];
+  mkMailPasswordSecret = name: {
+    name  = "mailserver/${name}/hashedPassword";
+    value = {};
+  };
+  mkMailLoginAccount = name: {
+    inherit name;
+    value = {
+      hashedPasswordFile = config.sops.secrets."mailserver/${name}/hashedPassword".path;
+    };
+  };
   sslOpts = {
     sslCertificate    = config.sops.secrets."ssl/porkbun/${domain}/domain.cert.pem".path;
     sslCertificateKey = config.sops.secrets."ssl/porkbun/${domain}/private.key.pem".path;
@@ -91,6 +111,24 @@ in {
     gnupg.sshKeyPaths  = [ ];
     age.sshKeyPaths    = [ "/etc/ssh/ssh_host_ed25519_key" ];
     defaultSopsFile    = "${flake}/sus/hectic-lab.yaml";
+    secrets = builtins.listToAttrs (map mkMailPasswordSecret mailUserNames) // {
+      "init-postgresql" = {
+        key = "init-postgresql";
+      };
+      "ssl/porkbun/${domain}/domain.cert.pem" = {
+        group = "nginx";
+        mode  = "0440";
+      };
+      "ssl/porkbun/${domain}/private.key.pem" = {
+        group = "nginx";
+        mode  = "0440";
+      };
+      "ssl/porkbun/${domain}/public.key.pem" = {
+        group = "nginx";
+        mode  = "0440";
+      };
+      "wg-bfs/private-key" = {};
+    };
   };
 
   users.users.root.openssh.authorizedKeys.keys = [
@@ -120,43 +158,10 @@ in {
     ];
   };
 
-  sops.secrets."mailserver/security/hashedPassword"      = {};
-  sops.secrets."mailserver/yukkop/hashedPassword"        = {};
-  sops.secrets."mailserver/daniil-perlyk/hashedPassword" = {};
-  sops.secrets."mailserver/snuff/hashedPassword"         = {};
-  sops.secrets."mailserver/antoshka/hashedPassword"      = {};
-  sops.secrets."mailserver/founders/hashedPassword"      = {};
-  sops.secrets."mailserver/lvgkcfjl/hashedPassword"      = {};
-  sops.secrets."init-postgresql" = {
-    key = "init-postgresql";
-  };
-
   services.mailserver = {
     enable = true;
     domain = domain;
-    loginAccounts = {
-      "security" = {
-        hashedPasswordFile = config.sops.secrets."mailserver/security/hashedPassword".path;
-      };
-      "founders" = {
-        hashedPasswordFile = config.sops.secrets."mailserver/founders/hashedPassword".path;
-      };
-      "lvgkcfjl" = {
-        hashedPasswordFile = config.sops.secrets."mailserver/lvgkcfjl/hashedPassword".path;
-      };
-      "yukkop" = {
-        hashedPasswordFile = config.sops.secrets."mailserver/yukkop/hashedPassword".path;
-      };
-      "daniil-perlyk" = {
-        hashedPasswordFile = config.sops.secrets."mailserver/daniil-perlyk/hashedPassword".path;
-      };
-      "snuff" = {
-        hashedPasswordFile = config.sops.secrets."mailserver/snuff/hashedPassword".path;
-      };
-      "antoshka" = {
-        hashedPasswordFile = config.sops.secrets."mailserver/antoshka/hashedPassword".path;
-      };
-    };
+    loginAccounts = builtins.listToAttrs (map mkMailLoginAccount mailUserNames);
   };
 
   mailserver.stateVersion = 3;
@@ -196,10 +201,6 @@ in {
     "d /var/www/store 0755 nginx nginx -"
   ];
 
-  sops.secrets."ssl/porkbun/${domain}/domain.cert.pem" = { group = "nginx"; mode = "0440"; };
-  sops.secrets."ssl/porkbun/${domain}/private.key.pem" = { group = "nginx"; mode = "0440"; };
-  sops.secrets."ssl/porkbun/${domain}/public.key.pem"  = { group = "nginx"; mode = "0440"; };
-
   services.nginx = {
     enable = true;
     # NOTE(yukkop): virtualHosts.${domain} is owned by the hectic-landing module
@@ -257,8 +258,9 @@ in {
       enable = true;
       settings.service.DISABLE_REGISTRATION = false;
       settings.server = {
-        HTTP_PORT = 11011;
-        SSH_PORT = 11012;
+        HTTP_PORT  = 11011;
+        #SSH_PORT   = 11012;
+        SSH_DOMAIN = "hectic-lab.com";
       };
       database = {
         createDatabase = true;
@@ -272,8 +274,6 @@ in {
 
   # === WireGuard (disabled) ===
 
-  sops.secrets."wg-bfs/private-key" = {};
-
   # networking.wireguard.interfaces = let
   #   subnet            = "10.13.37";
   #   externalInterface = "eth0";

sus/hectic-lab.yaml

@@ -5,6 +5,8 @@ mailserver:
         hashedPassword: ENC[AES256_GCM,data:zjhCFkmmMzQHn09uRz1S4NTNU8hVRY5ZSYRHn6Gd0u09Fc7inNVSPrO+Br41UagPmv526w9MMQoIbV3RiJq6E/mfhAouqybYbQ==,iv:aVjn+/X2ESgZU7p7jETONaqtsD2/NAFOd7IIbunTRaI=,tag:kueml9QdQYVBceFMCgWoGA==,type:str]
     daniil-perlyk:
         hashedPassword: ENC[AES256_GCM,data:vlmz39OxJ0iavBYW1ij2pn/1wFOx+yBWR16ZCDlnGvc9/BNwd8vhAPJw7D+bujA89UMAvIC7ERiSNzR2uamp0+aH8gKyzchEXg==,iv:H1NVynfp5X18PcyR2/R4mNZ69KpcbzAc+D2akl3aAmE=,tag:YjYh/y5o3bkajWDsyctJUw==,type:str]
+    iana-perlyk:
+        hashedPassword: ENC[AES256_GCM,data:RkMAMTIBvI84nccOU69GZsx3iblDPm11SnTGjTqVBFWES24bXH4bcU3Xq7lszY5xcH0W93uFULqGI3wom+szA1XefiGcpKhR9Q==,iv:DjahflSSkOoJyjBAC19AMu+OqOkD3H9PwenE4ylI9W4=,tag:qbcNFnKo7nipC3POsAxtdQ==,type:str]
     founders:
         hashedPassword: ENC[AES256_GCM,data:E+Xu/Ul3rFLlYDD0LGGRnc6RADlWmXpOM2OfkZFSzAf5thm8leRi9x5sroP25cO8CcSyBcOiUCBv3RC5ooXklm4cmpOx8LA8ug==,iv:RLbU6SBHKOBRCNZ6naxQMaNjWZOlNui6OaaVM2QkdZs=,tag:sO7CD+PVkdtvPvlUrpzW+g==,type:str]
     snuff:
@@ -75,7 +77,7 @@ sops:
             Yk43ZmlTc09aNFV1VjdjN2RWQlFWTDQKcYSvA2lHP8GS0lkYY19Tm8RXmFHQX5Ck
             qV2Fn22Fic4M5FVKDEMfaO6WmeXgki9a8dGeO9LlC+Phf16SOq7eLw==
             -----END AGE ENCRYPTED FILE-----
-    lastmodified: "2026-05-27T12:36:35Z"
-    mac: ENC[AES256_GCM,data:dqVqDqMRJFVhT78mO8q+X+Mf4TUqxY4ApOdkMAF9bvyvPAPW7kxbKEvn9H5LBDev9CNxfNF9siqaa7aEdATM6ylhrcWPdzPN04LorojdMNRQy/WFQ0rB1Lz1RrIvwltQD5K8RSPFCWWtF8rhqVTGdFafwCWpdmuKTx0HEPNXsPY=,iv:cWHx326j4aycz17N+q2NNqq0VwfeupmOiPdAtZRz7ws=,tag:LOKuvivMogAGQoVlSTu/vg==,type:str]
+    lastmodified: "2026-05-31T17:04:00Z"
+    mac: ENC[AES256_GCM,data:wkOHWvNXw3Cfh5ettROAstBdqNj2q5vna4LyOrhkyko4x4U+RhMDghUhtVy/AS48hX2yf5w13eIub/+10jm+HGxw70KA9LK9JuqsjdX8/IpJgWD52/ueZu+iO+dGGCXHLtu2ufWXwuqYAF0ea2sfQygChfhBclgNXYdmnJeOycQ=,iv:/i/0OGCziRYZodT0+dP72wzPV6lsKjieJavp/9LQXj0=,tag:OsM0Qy9o0wzmcgCNr6Bo0Q==,type:str]
     unencrypted_suffix: _unencrypted
     version: 3.10.2