diff --git a/docs/README.md b/docs/README.md index 611ae6c1..30900f7b 100644 --- a/docs/README.md +++ b/docs/README.md @@ -1,3 +1,23 @@ # Documentation - [Using the `hectic` Attic Cache](./attic-cache.md) + +## Gitea runner labels + +Common labels for zero-idle runners: + +- `ubuntu-latest` — default Ubuntu-style smoke / host runner path +- `nix` — Nix-capable runner path +- `gross-x86` — x86 fallback chain `cx53` / `cx43` / `cx33` +- `gross-arm` — ARM fallback chain `cax41` / `cax31` / `cax21` +- `gross-x86-perf` — x86 performance chain `cx53` / `cpx62` / `cpx52` +- `gross-mixed-econ` — mixed chain `cx53` / `cax41` / `cx43` +- `gross-nix-x86` — `gross-x86` + Nix bootstrap +- `gross-nix-arm` — `gross-arm` + Nix bootstrap +- `gross-nix-x86-perf` — `gross-x86-perf` + Nix bootstrap +- `gross-nix-mixed-econ` — `gross-mixed-econ` + Nix bootstrap + +Region order for fallback: `nbg1`, then `fsn1`, then `hel1`. + +Operational details: `infra/gitea-runners/runbook.md` and +`package/gitea-runner-controller/decide.sh`. diff --git a/nixos/module/hectic/service/gitea-runner-controller.nix b/nixos/module/hectic/service/gitea-runner-controller.nix index 68c15c38..718ed9cd 100644 --- a/nixos/module/hectic/service/gitea-runner-controller.nix +++ b/nixos/module/hectic/service/gitea-runner-controller.nix @@ -49,10 +49,12 @@ let "GCR_SSH_PRIVKEY_FILE=${config.sops.secrets."${secretPrefix}/ssh-private-key".path}" "GCR_NIX_VERSION=${cfg.nixVersion}" "GCR_NIX_TARBALL_SHA256=${cfg.nixTarballSha256}" + "GCR_ARM_NIX_TARBALL_SHA256=${cfg.armNixTarballSha256}" "GCR_ACT_RUNNER_VERSION=${cfg.actRunnerVersion}" "GCR_ACT_RUNNER_SHA256=${cfg.actRunnerSha256}" ] - ++ lib.optionals (cfg.imageId != null) [ "GCR_IMAGE_ID=${cfg.imageId}" ]; + ++ lib.optionals (cfg.imageId != null) [ "GCR_IMAGE_ID=${cfg.imageId}" ] + ++ lib.optionals (cfg.armImageId != null) [ "GCR_ARM_IMAGE_ID=${cfg.armImageId}" ]; in { options = { @@ -118,6 +120,15 @@ in creation while null. ''; }; + armImageId = lib.mkOption { + type = with lib.types; nullOr str; + default = null; + example = "423979717"; + description = '' + Hetzner ARM image/snapshot id for ephemeral VMs. Required for labels + whose fallback chain includes ARM server types. + ''; + }; actRunnerVersion = lib.mkOption { type = lib.types.str; default = "1.0.6"; @@ -138,6 +149,11 @@ in default = "85d1847d06d5d56167796d3f61cd992908de84584db3e700da031a782b59ea22"; description = "sha256 of the pinned Nix x86_64-linux tarball, verified at bootstrap."; }; + armNixTarballSha256 = lib.mkOption { + type = lib.types.str; + default = "3dffb118772382e35526806fb97acc05df7ad6dc29dbe52b921b77e52e39f571"; + description = "sha256 of the pinned Nix aarch64-linux tarball, verified at bootstrap."; + }; debugSshPublicKey = lib.mkOption { type = with lib.types; nullOr str; default = null; diff --git a/nixos/system/hectic-lab/hectic-lab.nix b/nixos/system/hectic-lab/hectic-lab.nix index 1c841774..a9e2e976 100644 --- a/nixos/system/hectic-lab/hectic-lab.nix +++ b/nixos/system/hectic-lab/hectic-lab.nix @@ -110,6 +110,7 @@ in { # Runbook: infra/gitea-runners/runbook.md "Ephemeral VM runner cutover". enable = true; imageId = "424558114"; # MicroOS x86 + Hetzner datasource dhcpcd fix + armImageId = "423979717"; # OpenSUSE MicroOS ARM K3S 2026-08-24 snapshot allowedRepos = [ "hinterland/*" "yukkop/*" diff --git a/package/gitea-runner-controller/decide.sh b/package/gitea-runner-controller/decide.sh index 325f1b0d..bc2a228a 100644 --- a/package/gitea-runner-controller/decide.sh +++ b/package/gitea-runner-controller/decide.sh @@ -5,14 +5,108 @@ # gcr_decide LABEL REPO -> prints " " and # returns 0 when allowed; returns 1 with reason on stderr otherwise. -gcr_label_profile() { +gcr_server_hourly_rate() { case "$1" in - nix) printf 'cx33 180 0.008' ;; - ubuntu-latest) printf 'cx33 60 0.008' ;; + cx23) printf '0.004' ;; + cx33) printf '0.008' ;; + cx43) printf '0.016' ;; + cx53) printf '0.032' ;; + cax21) printf '0.003' ;; + cax31) printf '0.006' ;; + cax41) printf '0.012' ;; + cpx52) printf '0.036' ;; + cpx62) printf '0.072' ;; *) return 1 ;; esac } +gcr_label_ttl() { + case "$1" in + ubuntu-latest) printf '60' ;; + nix) printf '180' ;; + gross-x86) printf '180' ;; + gross-arm) printf '180' ;; + gross-x86-perf) printf '180' ;; + gross-mixed-econ) printf '180' ;; + gross-nix-x86) printf '180' ;; + gross-nix-arm) printf '180' ;; + gross-nix-x86-perf) printf '180' ;; + gross-nix-mixed-econ) printf '180' ;; + *) return 1 ;; + esac +} + +# gcr_label_candidates LABEL -> lines: " " +gcr_label_candidates() { + label="$1" + case "$label" in + ubuntu-latest) + printf '%s\n' 'cx33 nbg1 amd64' 'cx33 fsn1 amd64' 'cx33 hel1 amd64' + ;; + nix) + printf '%s\n' 'cx33 nbg1 amd64' 'cx33 fsn1 amd64' 'cx33 hel1 amd64' + ;; + gross-x86) + printf '%s\n' \ + 'cx53 nbg1 amd64' 'cx53 fsn1 amd64' 'cx53 hel1 amd64' \ + 'cx43 nbg1 amd64' 'cx43 fsn1 amd64' 'cx43 hel1 amd64' \ + 'cx33 nbg1 amd64' 'cx33 fsn1 amd64' 'cx33 hel1 amd64' + ;; + gross-arm) + printf '%s\n' \ + 'cax41 nbg1 arm64' 'cax41 fsn1 arm64' 'cax41 hel1 arm64' \ + 'cax31 nbg1 arm64' 'cax31 fsn1 arm64' 'cax31 hel1 arm64' \ + 'cax21 nbg1 arm64' 'cax21 fsn1 arm64' 'cax21 hel1 arm64' + ;; + gross-x86-perf) + printf '%s\n' \ + 'cx53 nbg1 amd64' 'cx53 fsn1 amd64' 'cx53 hel1 amd64' \ + 'cpx62 nbg1 amd64' 'cpx62 fsn1 amd64' 'cpx62 hel1 amd64' \ + 'cpx52 nbg1 amd64' 'cpx52 fsn1 amd64' 'cpx52 hel1 amd64' + ;; + gross-mixed-econ) + printf '%s\n' \ + 'cx53 nbg1 amd64' 'cx53 fsn1 amd64' 'cx53 hel1 amd64' \ + 'cax41 nbg1 arm64' 'cax41 fsn1 arm64' 'cax41 hel1 arm64' \ + 'cx43 nbg1 amd64' 'cx43 fsn1 amd64' 'cx43 hel1 amd64' + ;; + gross-nix-x86) + printf '%s\n' \ + 'cx53 nbg1 amd64' 'cx53 fsn1 amd64' 'cx53 hel1 amd64' \ + 'cx43 nbg1 amd64' 'cx43 fsn1 amd64' 'cx43 hel1 amd64' \ + 'cx33 nbg1 amd64' 'cx33 fsn1 amd64' 'cx33 hel1 amd64' + ;; + gross-nix-arm) + printf '%s\n' \ + 'cax41 nbg1 arm64' 'cax41 fsn1 arm64' 'cax41 hel1 arm64' \ + 'cax31 nbg1 arm64' 'cax31 fsn1 arm64' 'cax31 hel1 arm64' \ + 'cax21 nbg1 arm64' 'cax21 fsn1 arm64' 'cax21 hel1 arm64' + ;; + gross-nix-x86-perf) + printf '%s\n' \ + 'cx53 nbg1 amd64' 'cx53 fsn1 amd64' 'cx53 hel1 amd64' \ + 'cpx62 nbg1 amd64' 'cpx62 fsn1 amd64' 'cpx62 hel1 amd64' \ + 'cpx52 nbg1 amd64' 'cpx52 fsn1 amd64' 'cpx52 hel1 amd64' + ;; + gross-nix-mixed-econ) + printf '%s\n' \ + 'cx53 nbg1 amd64' 'cx53 fsn1 amd64' 'cx53 hel1 amd64' \ + 'cax41 nbg1 arm64' 'cax41 fsn1 arm64' 'cax41 hel1 arm64' \ + 'cx43 nbg1 amd64' 'cx43 fsn1 amd64' 'cx43 hel1 amd64' + ;; + *) return 1 ;; + esac +} + +gcr_label_profile() { + label="$1" + ttl="$(gcr_label_ttl "$label")" || return 1 + first="$(gcr_label_candidates "$label" | head -n1)" || return 1 + set -- $first + rate="$(gcr_server_hourly_rate "$1")" || return 1 + printf '%s %s %s' "$1" "$ttl" "$rate" +} + gcr_repo_allowed() { repo="$1" oldIFS="$IFS" diff --git a/package/gitea-runner-controller/default.nix b/package/gitea-runner-controller/default.nix index 76da5153..b58b37ff 100644 --- a/package/gitea-runner-controller/default.nix +++ b/package/gitea-runner-controller/default.nix @@ -24,6 +24,7 @@ let "SC2086" # word splitting on purpose: env lists and profile triples "SC2046" # same, command substitution into set -- "SC2329" + "SC2154" ]; runtimeDeps = [ diff --git a/package/gitea-runner-controller/hcloud.sh b/package/gitea-runner-controller/hcloud.sh index bb4ee15b..02a80af6 100644 --- a/package/gitea-runner-controller/hcloud.sh +++ b/package/gitea-runner-controller/hcloud.sh @@ -7,6 +7,14 @@ GCR_API="https://api.hetzner.cloud/v1" +gcr_image_id_for_arch() { + case "$1" in + amd64) [ -n "${GCR_IMAGE_ID:-}" ] && printf '%s' "$GCR_IMAGE_ID" ;; + arm64) [ -n "${GCR_ARM_IMAGE_ID:-}" ] && printf '%s' "$GCR_ARM_IMAGE_ID" ;; + *) return 1 ;; + esac +} + gcr_hcloud_token() { test -n "${HCLOUD_TOKEN_FILE:-}" && test -r "$HCLOUD_TOKEN_FILE" || { gcr_log error --ns=hcloud "HCLOUD_TOKEN_FILE missing or unreadable" @@ -154,49 +162,51 @@ gcr_vm_create() { vm_name="$1"; label="$2"; server_type="$3"; ttl_min="$4" reg_token="$5"; job_id="$6"; attempt="$7"; repo="$8" - test -n "${GCR_IMAGE_ID:-}" || { - gcr_log error --ns=hcloud "GCR_IMAGE_ID not set; refusing VM creation" - return 1 - } - - userdata="$(gcr_vm_build_userdata "$vm_name" "$label" "$reg_token")" - payload="$(jq -n \ - --arg name "$vm_name" \ - --arg stype "$server_type" \ - --arg image "$GCR_IMAGE_ID" \ - --arg loc "${GCR_HETZNER_LOCATION:-nbg1}" \ - --arg udata "$userdata" \ - --arg jid "$job_id" \ - --arg att "$attempt" \ - --arg repo "$repo" \ - --arg label "$label" \ - --arg ts "$(date -u '+%s')" \ - --arg ttl "$ttl_min" \ - --arg repo_safe "$(printf '%s' "$repo" | tr '/:' '--')" \ - '{name:$name, server_type:$stype, image:$image, location:$loc, - start_after_create:true, - labels:{ - "gitea-runner-controller":"managed", - "gcr.job-id":$jid, "gcr.run-attempt":$att, - "gcr.repo":$repo_safe, "gcr.label":$label, - "gcr.created-at":$ts, "gcr.ttl-min":$ttl}}')" - - # Hetzner placement is occasionally transient (resource_unavailable); - # retry a few times before giving up. NOTE: userdata/cloud-init is NOT - # used — bootstrap happens over SSH from the controller (see - # gcr_vm_bootstrap_ssh); MicroOS snapshot's Hetzner datasource cannot - # fetch user-data (DHCP Exception on this image lineage). - attempt_n=0 - while :; do - attempt_n=$((attempt_n + 1)) + ttl_min="$(gcr_label_ttl "$label")" || return 1 + candidates="$(gcr_label_candidates "$label")" || return 1 + candidate_n=0 + while read -r candidate_type candidate_loc candidate_arch; do + [ -n "${candidate_type:-}" ] || continue + candidate_n=$((candidate_n + 1)) + image_id="$(gcr_image_id_for_arch "$candidate_arch")" || { + gcr_log warn --ns=hcloud "skip candidate[$candidate_n] label=$label arch=$candidate_arch no image" + continue + } + payload="$(jq -n \ + --arg name "$vm_name" \ + --arg stype "$candidate_type" \ + --arg image "$image_id" \ + --arg loc "$candidate_loc" \ + --arg jid "$job_id" \ + --arg att "$attempt" \ + --arg repo "$repo" \ + --arg label "$label" \ + --arg arch "$candidate_arch" \ + --arg ts "$(date -u '+%s')" \ + --arg ttl "$ttl_min" \ + --arg repo_safe "$(printf '%s' "$repo" | tr '/:' '--')" \ + '{name:$name, server_type:$stype, image:$image, location:$loc, + start_after_create:true, + labels:{ + "gitea-runner-controller":"managed", + "gcr.job-id":$jid, "gcr.run-attempt":$att, + "gcr.repo":$repo_safe, "gcr.label":$label, + "gcr.arch":$arch, "gcr.location":$loc, + "gcr.created-at":$ts, "gcr.ttl-min":$ttl}}')" + gcr_log info --ns=hcloud "try candidate[$candidate_n] label=$label type=$candidate_type arch=$candidate_arch loc=$candidate_loc" if gcr_hcloud_req POST /servers "$payload"; then jq -r '.server.id' "$GCR_LAST_BODY" return 0 fi - gcr_log warn --ns=hcloud "create attempt=$attempt_n failed" - [ "$attempt_n" -ge 3 ] && return 1 - sleep $((attempt_n * 10)) - done + if [ "$candidate_n" -le 3 ]; then + sleep 5 + else + sleep 1 + fi + done < /usr/local/sbin/gcr-install <&2 + exit 1 + ;; + esac + curl -fsSL "https://releases.nixos.org/nix/nix-$GCR_NIX_VERSION/nix-$GCR_NIX_VERSION-$nix_arch.tar.xz" -o /tmp/nix.tar.xz + printf '%s /tmp/nix.tar.xz\n' "$nix_sha" | sha256sum -c - tar -xJf /tmp/nix.tar.xz -C /tmp - /tmp/nix-$GCR_NIX_VERSION-x86_64-linux/install --no-daemon + /tmp/nix-$GCR_NIX_VERSION-$nix_arch/install --no-daemon rm -rf /tmp/nix* -fi + ;; +esac curl -fsSL "https://dl.gitea.com/gitea-runner/$GCR_ACT_RUNNER_VERSION/gitea-runner-$GCR_ACT_RUNNER_VERSION-linux-amd64" -o /usr/local/bin/gitea-runner printf '%s /usr/local/bin/gitea-runner\n' "$GCR_ACT_RUNNER_SHA256" | sha256sum -c - chmod 0755 /usr/local/bin/gitea-runner