gross

devshell/gitea-runners.nix

@@ -43,7 +43,8 @@
     trap 'rm -rf "$workdir"' EXIT HUP INT TERM
     ssh_private_key_file="$workdir/packer-ssh-key"
     ssh-keygen -q -t ed25519 -N "" -f "$ssh_private_key_file"
-    image_public_key_file="''${GITEA_RUNNERS_IMAGE_SSH_PUBLIC_KEY_FILE:-$HOME/.ssh/id_ed25519.pub}"
+    : "''${GITEA_RUNNERS_IMAGE_SSH_PUBLIC_KEY_FILE:?GITEA_RUNNERS_IMAGE_SSH_PUBLIC_KEY_FILE is not set}"
+    image_public_key_file="$GITEA_RUNNERS_IMAGE_SSH_PUBLIC_KEY_FILE"
     test -r "$image_public_key_file"
     packer_public_key_b64="$(base64 -w0 "$ssh_private_key_file.pub")"
     image_public_key_b64="$(base64 -w0 "$image_public_key_file")"
@@ -105,9 +106,12 @@
    test -n "$root_device"
    mount -o subvol=@ "$root_device" /mnt
    install -d -m 0700 /mnt/root/.ssh
-  printf '%s' '__IMAGE_PUBLIC_KEY_B64__' | base64 -d > /mnt/root/.ssh/authorized_keys
-  printf '\n%s' '__PACKER_PUBLIC_KEY_B64__' | base64 -d >> /mnt/root/.ssh/authorized_keys
-  chmod 0600 /mnt/root/.ssh/authorized_keys
+   printf '%s' '__IMAGE_PUBLIC_KEY_B64__' | base64 -d > /mnt/root/.ssh/gcr_authorized_keys
+   printf '\n%s' '__PACKER_PUBLIC_KEY_B64__' | base64 -d >> /mnt/root/.ssh/gcr_authorized_keys
+   chmod 0600 /mnt/root/.ssh/gcr_authorized_keys
+   install -d -m 0755 /mnt/etc/ssh/sshd_config.d
+   printf '%s\n' 'PermitRootLogin prohibit-password' 'PubkeyAuthentication yes' 'AuthorizedKeysFile .ssh/authorized_keys .ssh/gcr_authorized_keys' > /mnt/etc/ssh/sshd_config.d/99-gcr-root.conf
+   chmod 0644 /mnt/etc/ssh/sshd_config.d/99-gcr-root.conf
    sync
    umount /mnt
 EOF
@@ -116,9 +120,15 @@ EOF
     sed -i "s|__IMAGE_PUBLIC_KEY_B64__|$image_public_key_b64|" \
       "$workdir/image-key-injection.txt"
     awk -v inject_file="$workdir/image-key-injection.txt" \
-      '/done[.] Rebooting/ {
+      '/rm -f \/root\/.ssh\/authorized_keys/ {
+         cleanup=1
+       }
+       cleanup && /sleep 1/ {
+         print
          while ((getline line < inject_file) > 0) print line
          close(inject_file)
+         cleanup=0
+         next
        }
       { print }' \
       "$packer_dir/hcloud-microos-snapshots.pkr.hcl" \

nixos/module/hectic/service/gitea-runner-controller.nix

@@ -47,12 +47,14 @@ let
       "GITEA_REGISTRATION_TOKEN_FILE=${registrationTokenPath}"
       "GITEA_ADMIN_TOKEN_FILE=${config.sops.secrets."${secretPrefix}/admin-token".path}"
       "GCR_SSH_PRIVKEY_FILE=${config.sops.secrets."${secretPrefix}/ssh-private-key".path}"
+      "GCR_HCLOUD_SSH_KEY_ID=${toString cfg.hcloudSshKeyId}"
       "GCR_NIX_VERSION=${cfg.nixVersion}"
       "GCR_NIX_TARBALL_SHA256=${cfg.nixTarballSha256}"
       "GCR_ARM_NIX_TARBALL_SHA256=${cfg.armNixTarballSha256}"
       "GCR_ACT_RUNNER_VERSION=${cfg.actRunnerVersion}"
       "GCR_ACT_RUNNER_SHA256=${cfg.actRunnerSha256}"
     ]
+    ++ lib.optional (cfg.debugSshPublicKey != null) "\"GCR_DEBUG_SSH_PUBKEY=${cfg.debugSshPublicKey}\""
     ++ lib.optionals (cfg.imageId != null) [ "GCR_IMAGE_ID=${cfg.imageId}" ]
     ++ lib.optionals (cfg.armImageId != null) [ "GCR_ARM_IMAGE_ID=${cfg.armImageId}" ]
     ++ lib.optionals (cfg.nixImageId != null) [ "GCR_NIX_IMAGE_ID=${cfg.nixImageId}" ]
@@ -177,6 +179,11 @@ in
           used by gcr_bootstrap_script documentation).
         '';
       };
+      hcloudSshKeyId = lib.mkOption {
+        type = lib.types.nullOr lib.types.ints.positive;
+        default = null;
+        description = "Hetzner project SSH key ID injected into ephemeral VMs at creation.";
+      };
       bootstrapSshPrivateKeyFile = lib.mkOption {
         type = lib.types.path;
         description = ''

nixos/system/hectic-lab/hectic-lab.nix

@@ -109,7 +109,7 @@ in {
       # NOTE(yukkop): ephemeral Hetzner VM runners (1 VM = 1 job).
       # Runbook: infra/gitea-runners/runbook.md "Ephemeral VM runner cutover".
       enable  = true;
-      imageId = "424558114"; # MicroOS x86 + Hetzner datasource dhcpcd fix
+      imageId = "429564260"; # MicroOS x86 + persistent controller SSH key
       armImageId = "423979717"; # OpenSUSE MicroOS ARM K3S 2026-08-24 snapshot
       nixImageId = "161547269"; # Ubuntu 24.04 x86; Nix needs writable root
       armNixImageId = "161547270"; # Ubuntu 24.04 ARM; Nix needs writable root
@@ -119,7 +119,8 @@ in {
         "hectic-lab/*"
       ];
       # FIXME(yukkop): debug key for bootstrap debugging; remove once E2E stable.
-      debugSshPublicKey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIJBLxMo5icX2Xyng7mcWGnIi+c4ZbVygjPhuU8noCkfZ yukkop@nixos";
+    debugSshPublicKey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIBSWIv80pyCMDQ6zH34P2qWizpOcO7X86BVhMGtbob9U gcr-controller@hectic-lab";
+    hcloudSshKeyId = 118512401;
     };
   };
 

package/gitea-runner-controller/hcloud.sh

@@ -10,7 +10,7 @@ GCR_API="https://api.hetzner.cloud/v1"
 gcr_image_id_for_arch() {
     arch="$1"; label="${2:-}"
     nix_image=0
-    case "$label" in nix|gross-nix-*) nix_image=1 ;; esac
+    case "$label" in nix) nix_image=1 ;; esac
     case "$arch:$nix_image" in
         amd64:0) [ -n "${GCR_IMAGE_ID:-}" ] && printf '%s' "$GCR_IMAGE_ID" ;;
         arm64:0) [ -n "${GCR_ARM_IMAGE_ID:-}" ] && printf '%s' "$GCR_ARM_IMAGE_ID" ;;
@@ -189,9 +189,11 @@ gcr_vm_create() {
             --arg arch "$candidate_arch" \
             --arg ts "$(date -u '+%s')" \
              --arg ttl "$ttl_min" \
+             --arg ssh_key_id "${GCR_HCLOUD_SSH_KEY_ID:-}" \
             --arg repo_safe "$(printf '%s' "$repo" | tr '/:' '--')" \
             '{name:$name, server_type:$stype, image:$image, location:$loc,
                start_after_create:true,
+               ssh_keys:(if $ssh_key_id == "" then [] else [$ssh_key_id | tonumber] end),
               labels:{
                 "gitea-runner-controller":"managed",
                 "gcr.job-id":$jid, "gcr.run-attempt":$att,

sus/gitea-runners.yaml

@@ -5,7 +5,7 @@ gitea:
             hcloud-token: ENC[AES256_GCM,data:mw2P6v1k1MOtIpuARG7++T3ica8gY7HdfGOlzr9DyMzqDl7hGAYmpLitVDu++5mZGHZoVboH9YzVwU9d6frGhg==,iv:db2Dy8fZ3vaOlvT5QAxptBY/h2glIroPQq2gpQVIno0=,tag:haR1YKTegFQytkQ6ygAKgw==,type:str]
             webhook-secret: ENC[AES256_GCM,data:NppDTo0T+DDlCaNp6mcQWgGfaygFGwDR5bd6oylxFIk/XhSz6ATUpNWSrn9oZm2geuc/BHgGAwcgQU1LnhnJGw==,iv:zhwOxR6Pf/NKSiDME/ipNBhcDize/t4WcSob13QmjsA=,tag:FXJflbfmpgqKZuiEYwBcrg==,type:str]
             admin-token: ENC[AES256_GCM,data:inIN+ISgS+HgCEu/r9lfjRbbbCz83NCcwoSaeeqJys6JocEGBy0law==,iv:Lj6f3flrQWatHHy9diMQQmFsnSU0ckqoZB0AX6eL00U=,tag:OazXKYMPbYsy+GE2eVCaNQ==,type:str]
-            ssh-private-key: ENC[AES256_GCM,data:+RH7sfrKpFo+lVgTunc+qWrqKgbwWHxRDt5iXySLQCeNQR/Qr/S9NWnJgDA4iwZqXxeDq/WlMpKMLfPu7lGeVuMr0sp3D9BCljtRWeFp3/OsWeOaDjHjIxiQF+E6XeaU52lwDaBb3xTUV+4iWZVYHytEtaKkSplxDkNqa+AOxlJ6q5xK1KALJBajR6AhTiLt/IrSDrBCP2Sd79wiBaulcD+WOem25mm8SzbSbnVZ1NnSjvh5O6pKKmRho9yjPVj9yXPad3Hpder986JZSH/wte9koNAaBzEjtAPZd4Do00xboV0PMgFm2aTft2nbDbnpxDMN0wNbcpOmOmGzraxRRTFs8o/UHRsDnWw8ZZGTiqqivCt9q0cB5mVJSxaQpJlzqCX3hjPz8sbCdw/vzIQc76fQHqHcleeb1oht+2E6T2OdRStEDI2u1ts+yF0d2gk/99wYaMK4Oun7qCyffHKsFt/vb1tAT3IXNW0zgkYWJhBd3oUsbV16ImL82VZSDyjIwBFwORkOU6PIUWuqVIo=,iv:k4dQC5FTc1k01z33gVXPTna63l+Ixr1aW7/OXGt7R/A=,tag:/dRO7wRjyYTR3p+6H17w+Q==,type:str]
+            ssh-private-key: ENC[AES256_GCM,data:bV/9wtBE4YEB3HpryNJP11i0i61ij3kSXTNSCSoOPVerHnTV3hiETNHRZEwzjepaLfBeM4V/k1GYxI7kjcP53Emt0Qp1DLxU5e+G0htiLfk2ZYIBEYcVx7gK30PmTt2BD+pwsY2uz66gdbiWo68R3FuYoW57HHi5hRXMT39DgilCBDnYPQvDWPhw2ZKJMiGF9TixM9f8VLrmZf0JUm/xAi5QJF36C+rmcEhyBh8le4Q1c+J+ZWDjPDGMVwSMQWXbjBZvd+9H2pfbt3GfbNaPKUppuX99pc1uJmB2JBC1iKTAdt2oeYyqYoBXV3zoILcdGqGkceSoZOYVOwN0yixVHuVELNDnISMVidcu5Od/vS0g9uyvF6f91nKI9kjkri9xJZSwM8u/XUw00zFuQaI5eDpYTDWDWETvCeAWdYiHfnJ56DoZsCCS4eiImEyRKRZiXFCeKurkWPJsEbvmyia2tr/i0jygvhTP40qqy6InOZc09STL0mHDAszokO7eMRH5bTOUtq0Oa/nSLtEIEzubpDYPLCFMxm+nl3O7Vrkuu9eERjI=,iv:KegHSZCKUGOb71Lt6LVftOmjQwMhRhKVqit6m5atMUs=,tag:WlR2eVMsENwF4k78OShvxw==,type:str]
 hetzner:
     hcloud-token: ENC[AES256_GCM,data:cYnl+xaIvSikhuCM9amnPJQLrX/nmIqaxn9KQ9niH52/n43kH7KC4sLuuyFj7/LfBA2EJI+3PLjsH4jm1leIcQ==,iv:Sysmgajtf9miAlJPth1TIXVEKG6ehc6NRcEyXezqblU=,tag:5+m9bsdBKeK/v/KG9XRpWw==,type:str]
 s3:
@@ -58,7 +58,7 @@ sops:
             UWkwVjIwTTlZaERkTUptQ2tSa29zdDQKgx7e5FQiV6fJQz8S91nKRX3m2pE04+0P
             MEP9+q3RRmrGG685/WTH8O/m/fVQx2yQ/QgJ8YBrEyVAq4jqDumAmg==
             -----END AGE ENCRYPTED FILE-----
-    lastmodified: "2026-08-27T07:26:16Z"
-    mac: ENC[AES256_GCM,data:H71K/HjlVCdaOWLcOsdHJo591aCr5+Fo5ObX2H7yNJAOaN5sE1HH/blz9koxivFEyr/yCr6a9cMZzzQoiZ1M4iFLwjiH/ZkLm44tZJsv9Dc+IJskNO+4Am+31OEgMQaf8ciVSJqn5PRZqPV0bg+OxznShxY/ClxCeIxNINGsvnw=,iv:+FK69Dm5XnPsu4CX+MFILA1Ag+SGJNBV0g1TiT7snRI=,tag:5OX0VVTipocpfjZZ+Q+pLA==,type:str]
+    lastmodified: "2026-09-08T12:14:15Z"
+    mac: ENC[AES256_GCM,data:rGOTdeXsY3x2ySe1zWO84efc8LFHbVWTNZOgudzcxgxYmGyVrAzdEnQ7mLUGtJnzJCOo1pesSWON3+iWcwCg789gJrAE4uRbQETtsIuOQ4E1QsjZEaO8vJI0W3ASOPotYxNOO5U90H2siOYS4xZLoSevbnydO9G/kkPBvyJXJm0=,iv:sAmEBa6ZEcn1W+Iy1tNY6EDn7deoiOIKWVlgUmVazV0=,tag:Pt76HdUgbMrTxlDzUFKw5Q==,type:str]
     unencrypted_suffix: _unencrypted
     version: 3.12.1