hinterland/hearth
2
2
Fork 1
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

gross
runner nix smoke / nix label and flake smoke (push) Failing after 8s
Details

Browse Source
This commit is contained in:
yukkop
2026-09-08 19:31:04 +00:00
parent 6f80f672f2
commit beeb73736c
5 changed files with 50 additions and 30 deletions
Download Patch File Download Diff File Expand all files Collapse all files
devshell/gitea-runners.nix
+32 -22
View File
@@ -43,7 +43,8 @@
trap 'rm -rf "$workdir"' EXIT HUP INT TERM trap 'rm -rf "$workdir"' EXIT HUP INT TERM
ssh_private_key_file="$workdir/packer-ssh-key" ssh_private_key_file="$workdir/packer-ssh-key"
ssh-keygen -q -t ed25519 -N "" -f "$ssh_private_key_file" ssh-keygen -q -t ed25519 -N "" -f "$ssh_private_key_file"
image_public_key_file="''${GITEA_RUNNERS_IMAGE_SSH_PUBLIC_KEY_FILE:-$HOME/.ssh/id_ed25519.pub}" : "''${GITEA_RUNNERS_IMAGE_SSH_PUBLIC_KEY_FILE:?GITEA_RUNNERS_IMAGE_SSH_PUBLIC_KEY_FILE is not set}"
image_public_key_file="$GITEA_RUNNERS_IMAGE_SSH_PUBLIC_KEY_FILE"
test -r "$image_public_key_file" test -r "$image_public_key_file"
packer_public_key_b64="$(base64 -w0 "$ssh_private_key_file.pub")" packer_public_key_b64="$(base64 -w0 "$ssh_private_key_file.pub")"
image_public_key_b64="$(base64 -w0 "$image_public_key_file")" image_public_key_b64="$(base64 -w0 "$image_public_key_file")"
@@ -93,33 +94,42 @@
"$packer_dir/hcloud-microos-snapshots.pkr.hcl" "$packer_dir/hcloud-microos-snapshots.pkr.hcl"
cat > "$workdir/image-key-injection.txt" <<'EOF' cat > "$workdir/image-key-injection.txt" <<'EOF'
partprobe /dev/sda || true partprobe /dev/sda || true
udevadm settle udevadm settle
root_device="" root_device=""
for candidate in /dev/sda[0-9]*; do for candidate in /dev/sda[0-9]*; do
if [ "$(blkid -s TYPE -o value "$candidate" 2>/dev/null || true)" = btrfs ]; then if [ "$(blkid -s TYPE -o value "$candidate" 2>/dev/null || true)" = btrfs ]; then
root_device="$candidate" root_device="$candidate"
break break
fi fi
done done
test -n "$root_device" test -n "$root_device"
mount -o subvol=@ "$root_device" /mnt mount -o subvol=@ "$root_device" /mnt
install -d -m 0700 /mnt/root/.ssh install -d -m 0700 /mnt/root/.ssh
printf '%s' '__IMAGE_PUBLIC_KEY_B64__' | base64 -d > /mnt/root/.ssh/authorized_keys printf '%s' '__IMAGE_PUBLIC_KEY_B64__' | base64 -d > /mnt/root/.ssh/gcr_authorized_keys
printf '\n%s' '__PACKER_PUBLIC_KEY_B64__' | base64 -d >> /mnt/root/.ssh/authorized_keys printf '\n%s' '__PACKER_PUBLIC_KEY_B64__' | base64 -d >> /mnt/root/.ssh/gcr_authorized_keys
chmod 0600 /mnt/root/.ssh/authorized_keys chmod 0600 /mnt/root/.ssh/gcr_authorized_keys
sync install -d -m 0755 /mnt/etc/ssh/sshd_config.d
umount /mnt printf '%s\n' 'PermitRootLogin prohibit-password' 'PubkeyAuthentication yes' 'AuthorizedKeysFile .ssh/authorized_keys .ssh/gcr_authorized_keys' > /mnt/etc/ssh/sshd_config.d/99-gcr-root.conf
chmod 0644 /mnt/etc/ssh/sshd_config.d/99-gcr-root.conf
sync
umount /mnt
EOF EOF
sed -i "s|__PACKER_PUBLIC_KEY_B64__|$packer_public_key_b64|" \ sed -i "s|__PACKER_PUBLIC_KEY_B64__|$packer_public_key_b64|" \
"$workdir/image-key-injection.txt" "$workdir/image-key-injection.txt"
sed -i "s|__IMAGE_PUBLIC_KEY_B64__|$image_public_key_b64|" \ sed -i "s|__IMAGE_PUBLIC_KEY_B64__|$image_public_key_b64|" \
"$workdir/image-key-injection.txt" "$workdir/image-key-injection.txt"
awk -v inject_file="$workdir/image-key-injection.txt" \ awk -v inject_file="$workdir/image-key-injection.txt" \
'/done[.] Rebooting/ { '/rm -f \/root\/.ssh\/authorized_keys/ {
while ((getline line < inject_file) > 0) print line cleanup=1
close(inject_file) }
} cleanup && /sleep 1/ {
print
while ((getline line < inject_file) > 0) print line
close(inject_file)
cleanup=0
next
}
{ print }' \ { print }' \
"$packer_dir/hcloud-microos-snapshots.pkr.hcl" \ "$packer_dir/hcloud-microos-snapshots.pkr.hcl" \
> "$packer_dir/hcloud-microos-snapshots.pkr.hcl.tmp" > "$packer_dir/hcloud-microos-snapshots.pkr.hcl.tmp"
nixos/module/hectic/service/gitea-runner-controller.nix
+7
View File
@@ -47,12 +47,14 @@ let
"GITEA_REGISTRATION_TOKEN_FILE=${registrationTokenPath}" "GITEA_REGISTRATION_TOKEN_FILE=${registrationTokenPath}"
"GITEA_ADMIN_TOKEN_FILE=${config.sops.secrets."${secretPrefix}/admin-token".path}" "GITEA_ADMIN_TOKEN_FILE=${config.sops.secrets."${secretPrefix}/admin-token".path}"
"GCR_SSH_PRIVKEY_FILE=${config.sops.secrets."${secretPrefix}/ssh-private-key".path}" "GCR_SSH_PRIVKEY_FILE=${config.sops.secrets."${secretPrefix}/ssh-private-key".path}"
"GCR_HCLOUD_SSH_KEY_ID=${toString cfg.hcloudSshKeyId}"
"GCR_NIX_VERSION=${cfg.nixVersion}" "GCR_NIX_VERSION=${cfg.nixVersion}"
"GCR_NIX_TARBALL_SHA256=${cfg.nixTarballSha256}" "GCR_NIX_TARBALL_SHA256=${cfg.nixTarballSha256}"
"GCR_ARM_NIX_TARBALL_SHA256=${cfg.armNixTarballSha256}" "GCR_ARM_NIX_TARBALL_SHA256=${cfg.armNixTarballSha256}"
"GCR_ACT_RUNNER_VERSION=${cfg.actRunnerVersion}" "GCR_ACT_RUNNER_VERSION=${cfg.actRunnerVersion}"
"GCR_ACT_RUNNER_SHA256=${cfg.actRunnerSha256}" "GCR_ACT_RUNNER_SHA256=${cfg.actRunnerSha256}"
] ]
++ lib.optional (cfg.debugSshPublicKey != null) "\"GCR_DEBUG_SSH_PUBKEY=${cfg.debugSshPublicKey}\""
++ lib.optionals (cfg.imageId != null) [ "GCR_IMAGE_ID=${cfg.imageId}" ] ++ lib.optionals (cfg.imageId != null) [ "GCR_IMAGE_ID=${cfg.imageId}" ]
++ lib.optionals (cfg.armImageId != null) [ "GCR_ARM_IMAGE_ID=${cfg.armImageId}" ] ++ lib.optionals (cfg.armImageId != null) [ "GCR_ARM_IMAGE_ID=${cfg.armImageId}" ]
++ lib.optionals (cfg.nixImageId != null) [ "GCR_NIX_IMAGE_ID=${cfg.nixImageId}" ] ++ lib.optionals (cfg.nixImageId != null) [ "GCR_NIX_IMAGE_ID=${cfg.nixImageId}" ]
@@ -177,6 +179,11 @@ in
used by gcr_bootstrap_script documentation). used by gcr_bootstrap_script documentation).
''; '';
}; };
hcloudSshKeyId = lib.mkOption {
type = lib.types.nullOr lib.types.ints.positive;
default = null;
description = "Hetzner project SSH key ID injected into ephemeral VMs at creation.";
};
bootstrapSshPrivateKeyFile = lib.mkOption { bootstrapSshPrivateKeyFile = lib.mkOption {
type = lib.types.path; type = lib.types.path;
description = '' description = ''
nixos/system/hectic-lab/hectic-lab.nix
+3 -2
View File
@@ -109,7 +109,7 @@ in {
# NOTE(yukkop): ephemeral Hetzner VM runners (1 VM = 1 job). # NOTE(yukkop): ephemeral Hetzner VM runners (1 VM = 1 job).
# Runbook: infra/gitea-runners/runbook.md "Ephemeral VM runner cutover". # Runbook: infra/gitea-runners/runbook.md "Ephemeral VM runner cutover".
enable = true; enable = true;
imageId = "424558114"; # MicroOS x86 + Hetzner datasource dhcpcd fix imageId = "429564260"; # MicroOS x86 + persistent controller SSH key
armImageId = "423979717"; # OpenSUSE MicroOS ARM K3S 2026-08-24 snapshot armImageId = "423979717"; # OpenSUSE MicroOS ARM K3S 2026-08-24 snapshot
nixImageId = "161547269"; # Ubuntu 24.04 x86; Nix needs writable root nixImageId = "161547269"; # Ubuntu 24.04 x86; Nix needs writable root
armNixImageId = "161547270"; # Ubuntu 24.04 ARM; Nix needs writable root armNixImageId = "161547270"; # Ubuntu 24.04 ARM; Nix needs writable root
@@ -119,7 +119,8 @@ in {
"hectic-lab/*" "hectic-lab/*"
]; ];
# FIXME(yukkop): debug key for bootstrap debugging; remove once E2E stable. # FIXME(yukkop): debug key for bootstrap debugging; remove once E2E stable.
debugSshPublicKey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIJBLxMo5icX2Xyng7mcWGnIi+c4ZbVygjPhuU8noCkfZ yukkop@nixos"; debugSshPublicKey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIBSWIv80pyCMDQ6zH34P2qWizpOcO7X86BVhMGtbob9U gcr-controller@hectic-lab";
hcloudSshKeyId = 118512401;
}; };
}; };
package/gitea-runner-controller/hcloud.sh
+5 -3
View File
@@ -10,7 +10,7 @@ GCR_API="https://api.hetzner.cloud/v1"
gcr_image_id_for_arch() { gcr_image_id_for_arch() {
arch="$1"; label="${2:-}" arch="$1"; label="${2:-}"
nix_image=0 nix_image=0
case "$label" in nix|gross-nix-*) nix_image=1 ;; esac case "$label" in nix) nix_image=1 ;; esac
case "$arch:$nix_image" in case "$arch:$nix_image" in
amd64:0) [ -n "${GCR_IMAGE_ID:-}" ] && printf '%s' "$GCR_IMAGE_ID" ;; amd64:0) [ -n "${GCR_IMAGE_ID:-}" ] && printf '%s' "$GCR_IMAGE_ID" ;;
arm64:0) [ -n "${GCR_ARM_IMAGE_ID:-}" ] && printf '%s' "$GCR_ARM_IMAGE_ID" ;; arm64:0) [ -n "${GCR_ARM_IMAGE_ID:-}" ] && printf '%s' "$GCR_ARM_IMAGE_ID" ;;
@@ -188,10 +188,12 @@ gcr_vm_create() {
--arg label "$label" \ --arg label "$label" \
--arg arch "$candidate_arch" \ --arg arch "$candidate_arch" \
--arg ts "$(date -u '+%s')" \ --arg ts "$(date -u '+%s')" \
--arg ttl "$ttl_min" \ --arg ttl "$ttl_min" \
--arg ssh_key_id "${GCR_HCLOUD_SSH_KEY_ID:-}" \
--arg repo_safe "$(printf '%s' "$repo" | tr '/:' '--')" \ --arg repo_safe "$(printf '%s' "$repo" | tr '/:' '--')" \
'{name:$name, server_type:$stype, image:$image, location:$loc, '{name:$name, server_type:$stype, image:$image, location:$loc,
start_after_create:true, start_after_create:true,
ssh_keys:(if $ssh_key_id == "" then [] else [$ssh_key_id | tonumber] end),
labels:{ labels:{
"gitea-runner-controller":"managed", "gitea-runner-controller":"managed",
"gcr.job-id":$jid, "gcr.run-attempt":$att, "gcr.job-id":$jid, "gcr.run-attempt":$att,
sus/gitea-runners.yaml
+3 -3
View File
@@ -5,7 +5,7 @@ gitea:
hcloud-token: ENC[AES256_GCM,data:mw2P6v1k1MOtIpuARG7++T3ica8gY7HdfGOlzr9DyMzqDl7hGAYmpLitVDu++5mZGHZoVboH9YzVwU9d6frGhg==,iv:db2Dy8fZ3vaOlvT5QAxptBY/h2glIroPQq2gpQVIno0=,tag:haR1YKTegFQytkQ6ygAKgw==,type:str] hcloud-token: ENC[AES256_GCM,data:mw2P6v1k1MOtIpuARG7++T3ica8gY7HdfGOlzr9DyMzqDl7hGAYmpLitVDu++5mZGHZoVboH9YzVwU9d6frGhg==,iv:db2Dy8fZ3vaOlvT5QAxptBY/h2glIroPQq2gpQVIno0=,tag:haR1YKTegFQytkQ6ygAKgw==,type:str]
webhook-secret: ENC[AES256_GCM,data:NppDTo0T+DDlCaNp6mcQWgGfaygFGwDR5bd6oylxFIk/XhSz6ATUpNWSrn9oZm2geuc/BHgGAwcgQU1LnhnJGw==,iv:zhwOxR6Pf/NKSiDME/ipNBhcDize/t4WcSob13QmjsA=,tag:FXJflbfmpgqKZuiEYwBcrg==,type:str] webhook-secret: ENC[AES256_GCM,data:NppDTo0T+DDlCaNp6mcQWgGfaygFGwDR5bd6oylxFIk/XhSz6ATUpNWSrn9oZm2geuc/BHgGAwcgQU1LnhnJGw==,iv:zhwOxR6Pf/NKSiDME/ipNBhcDize/t4WcSob13QmjsA=,tag:FXJflbfmpgqKZuiEYwBcrg==,type:str]
admin-token: ENC[AES256_GCM,data:inIN+ISgS+HgCEu/r9lfjRbbbCz83NCcwoSaeeqJys6JocEGBy0law==,iv:Lj6f3flrQWatHHy9diMQQmFsnSU0ckqoZB0AX6eL00U=,tag:OazXKYMPbYsy+GE2eVCaNQ==,type:str] admin-token: ENC[AES256_GCM,data:inIN+ISgS+HgCEu/r9lfjRbbbCz83NCcwoSaeeqJys6JocEGBy0law==,iv:Lj6f3flrQWatHHy9diMQQmFsnSU0ckqoZB0AX6eL00U=,tag:OazXKYMPbYsy+GE2eVCaNQ==,type:str]
ssh-private-key: ENC[AES256_GCM,data:+RH7sfrKpFo+lVgTunc+qWrqKgbwWHxRDt5iXySLQCeNQR/Qr/S9NWnJgDA4iwZqXxeDq/WlMpKMLfPu7lGeVuMr0sp3D9BCljtRWeFp3/OsWeOaDjHjIxiQF+E6XeaU52lwDaBb3xTUV+4iWZVYHytEtaKkSplxDkNqa+AOxlJ6q5xK1KALJBajR6AhTiLt/IrSDrBCP2Sd79wiBaulcD+WOem25mm8SzbSbnVZ1NnSjvh5O6pKKmRho9yjPVj9yXPad3Hpder986JZSH/wte9koNAaBzEjtAPZd4Do00xboV0PMgFm2aTft2nbDbnpxDMN0wNbcpOmOmGzraxRRTFs8o/UHRsDnWw8ZZGTiqqivCt9q0cB5mVJSxaQpJlzqCX3hjPz8sbCdw/vzIQc76fQHqHcleeb1oht+2E6T2OdRStEDI2u1ts+yF0d2gk/99wYaMK4Oun7qCyffHKsFt/vb1tAT3IXNW0zgkYWJhBd3oUsbV16ImL82VZSDyjIwBFwORkOU6PIUWuqVIo=,iv:k4dQC5FTc1k01z33gVXPTna63l+Ixr1aW7/OXGt7R/A=,tag:/dRO7wRjyYTR3p+6H17w+Q==,type:str] ssh-private-key: ENC[AES256_GCM,data:bV/9wtBE4YEB3HpryNJP11i0i61ij3kSXTNSCSoOPVerHnTV3hiETNHRZEwzjepaLfBeM4V/k1GYxI7kjcP53Emt0Qp1DLxU5e+G0htiLfk2ZYIBEYcVx7gK30PmTt2BD+pwsY2uz66gdbiWo68R3FuYoW57HHi5hRXMT39DgilCBDnYPQvDWPhw2ZKJMiGF9TixM9f8VLrmZf0JUm/xAi5QJF36C+rmcEhyBh8le4Q1c+J+ZWDjPDGMVwSMQWXbjBZvd+9H2pfbt3GfbNaPKUppuX99pc1uJmB2JBC1iKTAdt2oeYyqYoBXV3zoILcdGqGkceSoZOYVOwN0yixVHuVELNDnISMVidcu5Od/vS0g9uyvF6f91nKI9kjkri9xJZSwM8u/XUw00zFuQaI5eDpYTDWDWETvCeAWdYiHfnJ56DoZsCCS4eiImEyRKRZiXFCeKurkWPJsEbvmyia2tr/i0jygvhTP40qqy6InOZc09STL0mHDAszokO7eMRH5bTOUtq0Oa/nSLtEIEzubpDYPLCFMxm+nl3O7Vrkuu9eERjI=,iv:KegHSZCKUGOb71Lt6LVftOmjQwMhRhKVqit6m5atMUs=,tag:WlR2eVMsENwF4k78OShvxw==,type:str]
hetzner: hetzner:
hcloud-token: ENC[AES256_GCM,data:cYnl+xaIvSikhuCM9amnPJQLrX/nmIqaxn9KQ9niH52/n43kH7KC4sLuuyFj7/LfBA2EJI+3PLjsH4jm1leIcQ==,iv:Sysmgajtf9miAlJPth1TIXVEKG6ehc6NRcEyXezqblU=,tag:5+m9bsdBKeK/v/KG9XRpWw==,type:str] hcloud-token: ENC[AES256_GCM,data:cYnl+xaIvSikhuCM9amnPJQLrX/nmIqaxn9KQ9niH52/n43kH7KC4sLuuyFj7/LfBA2EJI+3PLjsH4jm1leIcQ==,iv:Sysmgajtf9miAlJPth1TIXVEKG6ehc6NRcEyXezqblU=,tag:5+m9bsdBKeK/v/KG9XRpWw==,type:str]
s3: s3:
@@ -58,7 +58,7 @@ sops:
UWkwVjIwTTlZaERkTUptQ2tSa29zdDQKgx7e5FQiV6fJQz8S91nKRX3m2pE04+0P UWkwVjIwTTlZaERkTUptQ2tSa29zdDQKgx7e5FQiV6fJQz8S91nKRX3m2pE04+0P
MEP9+q3RRmrGG685/WTH8O/m/fVQx2yQ/QgJ8YBrEyVAq4jqDumAmg== MEP9+q3RRmrGG685/WTH8O/m/fVQx2yQ/QgJ8YBrEyVAq4jqDumAmg==
-----END AGE ENCRYPTED FILE----- -----END AGE ENCRYPTED FILE-----
lastmodified: "2026-08-27T07:26:16Z" lastmodified: "2026-09-08T12:14:15Z"
mac: ENC[AES256_GCM,data:H71K/HjlVCdaOWLcOsdHJo591aCr5+Fo5ObX2H7yNJAOaN5sE1HH/blz9koxivFEyr/yCr6a9cMZzzQoiZ1M4iFLwjiH/ZkLm44tZJsv9Dc+IJskNO+4Am+31OEgMQaf8ciVSJqn5PRZqPV0bg+OxznShxY/ClxCeIxNINGsvnw=,iv:+FK69Dm5XnPsu4CX+MFILA1Ag+SGJNBV0g1TiT7snRI=,tag:5OX0VVTipocpfjZZ+Q+pLA==,type:str] mac: ENC[AES256_GCM,data:rGOTdeXsY3x2ySe1zWO84efc8LFHbVWTNZOgudzcxgxYmGyVrAzdEnQ7mLUGtJnzJCOo1pesSWON3+iWcwCg789gJrAE4uRbQETtsIuOQ4E1QsjZEaO8vJI0W3ASOPotYxNOO5U90H2siOYS4xZLoSevbnydO9G/kkPBvyJXJm0=,iv:sAmEBa6ZEcn1W+Iy1tNY6EDn7deoiOIKWVlgUmVazV0=,tag:Pt76HdUgbMrTxlDzUFKw5Q==,type:str]
unencrypted_suffix: _unencrypted unencrypted_suffix: _unencrypted
version: 3.12.1 version: 3.12.1