diff --git a/devshell/gitea-runners.nix b/devshell/gitea-runners.nix index b8875563..514189cb 100644 --- a/devshell/gitea-runners.nix +++ b/devshell/gitea-runners.nix @@ -43,7 +43,8 @@ trap 'rm -rf "$workdir"' EXIT HUP INT TERM ssh_private_key_file="$workdir/packer-ssh-key" ssh-keygen -q -t ed25519 -N "" -f "$ssh_private_key_file" - image_public_key_file="''${GITEA_RUNNERS_IMAGE_SSH_PUBLIC_KEY_FILE:-$HOME/.ssh/id_ed25519.pub}" + : "''${GITEA_RUNNERS_IMAGE_SSH_PUBLIC_KEY_FILE:?GITEA_RUNNERS_IMAGE_SSH_PUBLIC_KEY_FILE is not set}" + image_public_key_file="$GITEA_RUNNERS_IMAGE_SSH_PUBLIC_KEY_FILE" test -r "$image_public_key_file" packer_public_key_b64="$(base64 -w0 "$ssh_private_key_file.pub")" image_public_key_b64="$(base64 -w0 "$image_public_key_file")" @@ -93,33 +94,42 @@ "$packer_dir/hcloud-microos-snapshots.pkr.hcl" cat > "$workdir/image-key-injection.txt" <<'EOF' - partprobe /dev/sda || true - udevadm settle - root_device="" - for candidate in /dev/sda[0-9]*; do - if [ "$(blkid -s TYPE -o value "$candidate" 2>/dev/null || true)" = btrfs ]; then - root_device="$candidate" - break - fi - done - test -n "$root_device" - mount -o subvol=@ "$root_device" /mnt - install -d -m 0700 /mnt/root/.ssh - printf '%s' '__IMAGE_PUBLIC_KEY_B64__' | base64 -d > /mnt/root/.ssh/authorized_keys - printf '\n%s' '__PACKER_PUBLIC_KEY_B64__' | base64 -d >> /mnt/root/.ssh/authorized_keys - chmod 0600 /mnt/root/.ssh/authorized_keys - sync - umount /mnt + partprobe /dev/sda || true + udevadm settle + root_device="" + for candidate in /dev/sda[0-9]*; do + if [ "$(blkid -s TYPE -o value "$candidate" 2>/dev/null || true)" = btrfs ]; then + root_device="$candidate" + break + fi + done + test -n "$root_device" + mount -o subvol=@ "$root_device" /mnt + install -d -m 0700 /mnt/root/.ssh + printf '%s' '__IMAGE_PUBLIC_KEY_B64__' | base64 -d > /mnt/root/.ssh/gcr_authorized_keys + printf '\n%s' '__PACKER_PUBLIC_KEY_B64__' | base64 -d >> /mnt/root/.ssh/gcr_authorized_keys + chmod 0600 /mnt/root/.ssh/gcr_authorized_keys + install -d -m 0755 /mnt/etc/ssh/sshd_config.d + printf '%s\n' 'PermitRootLogin prohibit-password' 'PubkeyAuthentication yes' 'AuthorizedKeysFile .ssh/authorized_keys .ssh/gcr_authorized_keys' > /mnt/etc/ssh/sshd_config.d/99-gcr-root.conf + chmod 0644 /mnt/etc/ssh/sshd_config.d/99-gcr-root.conf + sync + umount /mnt EOF sed -i "s|__PACKER_PUBLIC_KEY_B64__|$packer_public_key_b64|" \ "$workdir/image-key-injection.txt" sed -i "s|__IMAGE_PUBLIC_KEY_B64__|$image_public_key_b64|" \ "$workdir/image-key-injection.txt" awk -v inject_file="$workdir/image-key-injection.txt" \ - '/done[.] Rebooting/ { - while ((getline line < inject_file) > 0) print line - close(inject_file) - } + '/rm -f \/root\/.ssh\/authorized_keys/ { + cleanup=1 + } + cleanup && /sleep 1/ { + print + while ((getline line < inject_file) > 0) print line + close(inject_file) + cleanup=0 + next + } { print }' \ "$packer_dir/hcloud-microos-snapshots.pkr.hcl" \ > "$packer_dir/hcloud-microos-snapshots.pkr.hcl.tmp" diff --git a/nixos/module/hectic/service/gitea-runner-controller.nix b/nixos/module/hectic/service/gitea-runner-controller.nix index 23ec24e1..76aa15fc 100644 --- a/nixos/module/hectic/service/gitea-runner-controller.nix +++ b/nixos/module/hectic/service/gitea-runner-controller.nix @@ -47,12 +47,14 @@ let "GITEA_REGISTRATION_TOKEN_FILE=${registrationTokenPath}" "GITEA_ADMIN_TOKEN_FILE=${config.sops.secrets."${secretPrefix}/admin-token".path}" "GCR_SSH_PRIVKEY_FILE=${config.sops.secrets."${secretPrefix}/ssh-private-key".path}" + "GCR_HCLOUD_SSH_KEY_ID=${toString cfg.hcloudSshKeyId}" "GCR_NIX_VERSION=${cfg.nixVersion}" "GCR_NIX_TARBALL_SHA256=${cfg.nixTarballSha256}" "GCR_ARM_NIX_TARBALL_SHA256=${cfg.armNixTarballSha256}" "GCR_ACT_RUNNER_VERSION=${cfg.actRunnerVersion}" "GCR_ACT_RUNNER_SHA256=${cfg.actRunnerSha256}" ] + ++ lib.optional (cfg.debugSshPublicKey != null) "\"GCR_DEBUG_SSH_PUBKEY=${cfg.debugSshPublicKey}\"" ++ lib.optionals (cfg.imageId != null) [ "GCR_IMAGE_ID=${cfg.imageId}" ] ++ lib.optionals (cfg.armImageId != null) [ "GCR_ARM_IMAGE_ID=${cfg.armImageId}" ] ++ lib.optionals (cfg.nixImageId != null) [ "GCR_NIX_IMAGE_ID=${cfg.nixImageId}" ] @@ -177,6 +179,11 @@ in used by gcr_bootstrap_script documentation). ''; }; + hcloudSshKeyId = lib.mkOption { + type = lib.types.nullOr lib.types.ints.positive; + default = null; + description = "Hetzner project SSH key ID injected into ephemeral VMs at creation."; + }; bootstrapSshPrivateKeyFile = lib.mkOption { type = lib.types.path; description = '' diff --git a/nixos/system/hectic-lab/hectic-lab.nix b/nixos/system/hectic-lab/hectic-lab.nix index 6e4b5847..1b3b9aff 100644 --- a/nixos/system/hectic-lab/hectic-lab.nix +++ b/nixos/system/hectic-lab/hectic-lab.nix @@ -109,7 +109,7 @@ in { # NOTE(yukkop): ephemeral Hetzner VM runners (1 VM = 1 job). # Runbook: infra/gitea-runners/runbook.md "Ephemeral VM runner cutover". enable = true; - imageId = "424558114"; # MicroOS x86 + Hetzner datasource dhcpcd fix + imageId = "429564260"; # MicroOS x86 + persistent controller SSH key armImageId = "423979717"; # OpenSUSE MicroOS ARM K3S 2026-08-24 snapshot nixImageId = "161547269"; # Ubuntu 24.04 x86; Nix needs writable root armNixImageId = "161547270"; # Ubuntu 24.04 ARM; Nix needs writable root @@ -119,7 +119,8 @@ in { "hectic-lab/*" ]; # FIXME(yukkop): debug key for bootstrap debugging; remove once E2E stable. - debugSshPublicKey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIJBLxMo5icX2Xyng7mcWGnIi+c4ZbVygjPhuU8noCkfZ yukkop@nixos"; + debugSshPublicKey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIBSWIv80pyCMDQ6zH34P2qWizpOcO7X86BVhMGtbob9U gcr-controller@hectic-lab"; + hcloudSshKeyId = 118512401; }; }; diff --git a/package/gitea-runner-controller/hcloud.sh b/package/gitea-runner-controller/hcloud.sh index d6cd5c1b..0e5bd81d 100644 --- a/package/gitea-runner-controller/hcloud.sh +++ b/package/gitea-runner-controller/hcloud.sh @@ -10,7 +10,7 @@ GCR_API="https://api.hetzner.cloud/v1" gcr_image_id_for_arch() { arch="$1"; label="${2:-}" nix_image=0 - case "$label" in nix|gross-nix-*) nix_image=1 ;; esac + case "$label" in nix) nix_image=1 ;; esac case "$arch:$nix_image" in amd64:0) [ -n "${GCR_IMAGE_ID:-}" ] && printf '%s' "$GCR_IMAGE_ID" ;; arm64:0) [ -n "${GCR_ARM_IMAGE_ID:-}" ] && printf '%s' "$GCR_ARM_IMAGE_ID" ;; @@ -188,10 +188,12 @@ gcr_vm_create() { --arg label "$label" \ --arg arch "$candidate_arch" \ --arg ts "$(date -u '+%s')" \ - --arg ttl "$ttl_min" \ + --arg ttl "$ttl_min" \ + --arg ssh_key_id "${GCR_HCLOUD_SSH_KEY_ID:-}" \ --arg repo_safe "$(printf '%s' "$repo" | tr '/:' '--')" \ '{name:$name, server_type:$stype, image:$image, location:$loc, - start_after_create:true, + start_after_create:true, + ssh_keys:(if $ssh_key_id == "" then [] else [$ssh_key_id | tonumber] end), labels:{ "gitea-runner-controller":"managed", "gcr.job-id":$jid, "gcr.run-attempt":$att, diff --git a/sus/gitea-runners.yaml b/sus/gitea-runners.yaml index df4cfa85..793784ae 100644 --- a/sus/gitea-runners.yaml +++ b/sus/gitea-runners.yaml @@ -5,7 +5,7 @@ gitea: hcloud-token: ENC[AES256_GCM,data:mw2P6v1k1MOtIpuARG7++T3ica8gY7HdfGOlzr9DyMzqDl7hGAYmpLitVDu++5mZGHZoVboH9YzVwU9d6frGhg==,iv:db2Dy8fZ3vaOlvT5QAxptBY/h2glIroPQq2gpQVIno0=,tag:haR1YKTegFQytkQ6ygAKgw==,type:str] webhook-secret: ENC[AES256_GCM,data:NppDTo0T+DDlCaNp6mcQWgGfaygFGwDR5bd6oylxFIk/XhSz6ATUpNWSrn9oZm2geuc/BHgGAwcgQU1LnhnJGw==,iv:zhwOxR6Pf/NKSiDME/ipNBhcDize/t4WcSob13QmjsA=,tag:FXJflbfmpgqKZuiEYwBcrg==,type:str] admin-token: ENC[AES256_GCM,data:inIN+ISgS+HgCEu/r9lfjRbbbCz83NCcwoSaeeqJys6JocEGBy0law==,iv:Lj6f3flrQWatHHy9diMQQmFsnSU0ckqoZB0AX6eL00U=,tag:OazXKYMPbYsy+GE2eVCaNQ==,type:str] - ssh-private-key: ENC[AES256_GCM,data:+RH7sfrKpFo+lVgTunc+qWrqKgbwWHxRDt5iXySLQCeNQR/Qr/S9NWnJgDA4iwZqXxeDq/WlMpKMLfPu7lGeVuMr0sp3D9BCljtRWeFp3/OsWeOaDjHjIxiQF+E6XeaU52lwDaBb3xTUV+4iWZVYHytEtaKkSplxDkNqa+AOxlJ6q5xK1KALJBajR6AhTiLt/IrSDrBCP2Sd79wiBaulcD+WOem25mm8SzbSbnVZ1NnSjvh5O6pKKmRho9yjPVj9yXPad3Hpder986JZSH/wte9koNAaBzEjtAPZd4Do00xboV0PMgFm2aTft2nbDbnpxDMN0wNbcpOmOmGzraxRRTFs8o/UHRsDnWw8ZZGTiqqivCt9q0cB5mVJSxaQpJlzqCX3hjPz8sbCdw/vzIQc76fQHqHcleeb1oht+2E6T2OdRStEDI2u1ts+yF0d2gk/99wYaMK4Oun7qCyffHKsFt/vb1tAT3IXNW0zgkYWJhBd3oUsbV16ImL82VZSDyjIwBFwORkOU6PIUWuqVIo=,iv:k4dQC5FTc1k01z33gVXPTna63l+Ixr1aW7/OXGt7R/A=,tag:/dRO7wRjyYTR3p+6H17w+Q==,type:str] + ssh-private-key: ENC[AES256_GCM,data:bV/9wtBE4YEB3HpryNJP11i0i61ij3kSXTNSCSoOPVerHnTV3hiETNHRZEwzjepaLfBeM4V/k1GYxI7kjcP53Emt0Qp1DLxU5e+G0htiLfk2ZYIBEYcVx7gK30PmTt2BD+pwsY2uz66gdbiWo68R3FuYoW57HHi5hRXMT39DgilCBDnYPQvDWPhw2ZKJMiGF9TixM9f8VLrmZf0JUm/xAi5QJF36C+rmcEhyBh8le4Q1c+J+ZWDjPDGMVwSMQWXbjBZvd+9H2pfbt3GfbNaPKUppuX99pc1uJmB2JBC1iKTAdt2oeYyqYoBXV3zoILcdGqGkceSoZOYVOwN0yixVHuVELNDnISMVidcu5Od/vS0g9uyvF6f91nKI9kjkri9xJZSwM8u/XUw00zFuQaI5eDpYTDWDWETvCeAWdYiHfnJ56DoZsCCS4eiImEyRKRZiXFCeKurkWPJsEbvmyia2tr/i0jygvhTP40qqy6InOZc09STL0mHDAszokO7eMRH5bTOUtq0Oa/nSLtEIEzubpDYPLCFMxm+nl3O7Vrkuu9eERjI=,iv:KegHSZCKUGOb71Lt6LVftOmjQwMhRhKVqit6m5atMUs=,tag:WlR2eVMsENwF4k78OShvxw==,type:str] hetzner: hcloud-token: ENC[AES256_GCM,data:cYnl+xaIvSikhuCM9amnPJQLrX/nmIqaxn9KQ9niH52/n43kH7KC4sLuuyFj7/LfBA2EJI+3PLjsH4jm1leIcQ==,iv:Sysmgajtf9miAlJPth1TIXVEKG6ehc6NRcEyXezqblU=,tag:5+m9bsdBKeK/v/KG9XRpWw==,type:str] s3: @@ -58,7 +58,7 @@ sops: UWkwVjIwTTlZaERkTUptQ2tSa29zdDQKgx7e5FQiV6fJQz8S91nKRX3m2pE04+0P MEP9+q3RRmrGG685/WTH8O/m/fVQx2yQ/QgJ8YBrEyVAq4jqDumAmg== -----END AGE ENCRYPTED FILE----- - lastmodified: "2026-08-27T07:26:16Z" - mac: ENC[AES256_GCM,data:H71K/HjlVCdaOWLcOsdHJo591aCr5+Fo5ObX2H7yNJAOaN5sE1HH/blz9koxivFEyr/yCr6a9cMZzzQoiZ1M4iFLwjiH/ZkLm44tZJsv9Dc+IJskNO+4Am+31OEgMQaf8ciVSJqn5PRZqPV0bg+OxznShxY/ClxCeIxNINGsvnw=,iv:+FK69Dm5XnPsu4CX+MFILA1Ag+SGJNBV0g1TiT7snRI=,tag:5OX0VVTipocpfjZZ+Q+pLA==,type:str] + lastmodified: "2026-09-08T12:14:15Z" + mac: ENC[AES256_GCM,data:rGOTdeXsY3x2ySe1zWO84efc8LFHbVWTNZOgudzcxgxYmGyVrAzdEnQ7mLUGtJnzJCOo1pesSWON3+iWcwCg789gJrAE4uRbQETtsIuOQ4E1QsjZEaO8vJI0W3ASOPotYxNOO5U90H2siOYS4xZLoSevbnydO9G/kkPBvyJXJm0=,iv:sAmEBa6ZEcn1W+Iy1tNY6EDn7deoiOIKWVlgUmVazV0=,tag:Pt76HdUgbMrTxlDzUFKw5Q==,type:str] unencrypted_suffix: _unencrypted version: 3.12.1