From 0a2b96abc2cdae6540c8db0c10f97e0b73ab5448 Mon Sep 17 00:00:00 2001 From: yukkop Date: Thu, 10 Sep 2026 17:44:56 +0000 Subject: [PATCH] feat: highmem --- .gitea/workflows/deploy-neuro.yaml | 11 ++- infra/gitea-runners/runbook.md | 7 ++ package/gitea-runner-controller/decide.sh | 6 ++ package/gitea-runner-controller/hcloud.sh | 2 +- .../test/highmem-label.sh | 67 +++++++++++++++++++ test/package/with-attic-cache/time-budgets.py | 14 +++- 6 files changed, 103 insertions(+), 4 deletions(-) create mode 100644 test/package/gitea-runner-controller/test/highmem-label.sh diff --git a/.gitea/workflows/deploy-neuro.yaml b/.gitea/workflows/deploy-neuro.yaml index 4ed07fbf..3d254386 100644 --- a/.gitea/workflows/deploy-neuro.yaml +++ b/.gitea/workflows/deploy-neuro.yaml @@ -10,6 +10,15 @@ on: # yamllint disable-line rule:truthy required: false type: boolean default: true + runner_label: + description: Runner label for deploy job + required: false + type: choice + default: nix + options: + - nix + - gross-nix-x86-perf + - gross-nix-x86-highmem concurrency: group: deploy-neuro @@ -19,7 +28,7 @@ jobs: deploy: name: deploy neuro from performance node if: ${{ gitea.ref == 'refs/heads/master' }} - runs-on: gross-nix-x86-perf + runs-on: ${{ gitea.event.inputs.runner_label || 'nix' }} timeout-minutes: 435 env: NIX_CONFIG: | diff --git a/infra/gitea-runners/runbook.md b/infra/gitea-runners/runbook.md index 5280b92e..446a44f1 100644 --- a/infra/gitea-runners/runbook.md +++ b/infra/gitea-runners/runbook.md @@ -567,6 +567,7 @@ The `deploy-neuro` workflow uses these nested limits: | Final cache drain | 1 hour (`WITH_ATTIC_DRAIN_TIMEOUT=3600`) | | Workflow job | 435 minutes, including 15 minutes of setup/cleanup margin | | `gross-nix-x86-perf` runner | 480 minutes | +| `gross-nix-x86-highmem` runner | 480 minutes | | Gitea `actions.ENDLESS_TASK_TIMEOUT` | 8 hours | | VM hard lifetime from allocation | 480 minutes plus 10-minute controller grace | @@ -575,6 +576,12 @@ and Gitea watchdog settings before dispatching the longer workflow. Already allocated VMs retain the TTL and runner configuration assigned when they were created; updating the controller does not extend a running job. +`gross-nix-x86-highmem` is an explicit costly high-memory escape hatch backed +only by Hetzner CCX53 in `nbg1`, `fsn1`, or `hel1`; it may fall back by region +only, never to a lower-RAM server type. Current Hetzner public pricing for +Germany/Finland CCX53 is 0.8550 EUR/hour excluding IPv4, so one 480-minute +allocation reserves 6.84 EUR against the controller budget before VM creation. + These are maximum lifetimes: terminal jobs still trigger immediate VM teardown. The controller's budget reservation uses the full label TTL, so a long-running label reserves more of the existing monthly budget. Do not raise that budget or diff --git a/package/gitea-runner-controller/decide.sh b/package/gitea-runner-controller/decide.sh index f8ae1a69..14d6bf48 100644 --- a/package/gitea-runner-controller/decide.sh +++ b/package/gitea-runner-controller/decide.sh @@ -14,6 +14,7 @@ gcr_server_hourly_rate() { cax21) printf '0.003' ;; cax31) printf '0.006' ;; cax41) printf '0.012' ;; + ccx53) printf '0.8550' ;; cpx52) printf '0.036' ;; cpx62) printf '0.072' ;; *) return 1 ;; @@ -29,6 +30,7 @@ gcr_label_ttl() { gross-nix-x86) printf '180' ;; gross-nix-arm) printf '180' ;; gross-nix-x86-perf) printf '480' ;; + gross-nix-x86-highmem) printf '480' ;; gross-nix-mixed-econ) printf '180' ;; *) return 1 ;; esac @@ -80,6 +82,10 @@ gcr_label_candidates() { 'cpx62 nbg1 amd64' 'cpx62 fsn1 amd64' 'cpx62 hel1 amd64' \ 'cpx52 nbg1 amd64' 'cpx52 fsn1 amd64' 'cpx52 hel1 amd64' ;; + gross-nix-x86-highmem) + printf '%s\n' \ + 'ccx53 nbg1 amd64' 'ccx53 fsn1 amd64' 'ccx53 hel1 amd64' + ;; gross-nix-mixed-econ) printf '%s\n' \ 'cx53 nbg1 amd64' 'cx53 fsn1 amd64' 'cx53 hel1 amd64' \ diff --git a/package/gitea-runner-controller/hcloud.sh b/package/gitea-runner-controller/hcloud.sh index 2c070758..5ac75191 100644 --- a/package/gitea-runner-controller/hcloud.sh +++ b/package/gitea-runner-controller/hcloud.sh @@ -341,7 +341,7 @@ set -eu nix_arch="" nix_sha="" case "$label" in - nix|gross-nix-x86|gross-nix-arm|gross-nix-x86-perf|gross-nix-mixed-econ) + nix|gross-nix-*) case "\$(uname -m)" in x86_64) nix_arch=x86_64-linux diff --git a/test/package/gitea-runner-controller/test/highmem-label.sh b/test/package/gitea-runner-controller/test/highmem-label.sh new file mode 100644 index 00000000..18697254 --- /dev/null +++ b/test/package/gitea-runner-controller/test/highmem-label.sh @@ -0,0 +1,67 @@ +#!/bin/dash +set -eu + +. "$LOG_SH" +. "$STATE_SH" +. "$DECIDE_SH" +. "$HCLOUD_SH" + +gcr_state_init +export GCR_ALLOWED_REPOS='hectic-lab/util.nix' +export GCR_NIX_IMAGE_ID='424242' + +profile="$(gcr_decide gross-nix-x86-highmem hectic-lab/util.nix)" +test "$profile" = 'ccx53 480 0.8550' +test "$(gcr_label_ttl gross-nix-x86-highmem)" = '480' + +candidates="$(gcr_label_candidates gross-nix-x86-highmem)" +expected='ccx53 nbg1 amd64 +ccx53 fsn1 amd64 +ccx53 hel1 amd64' +test "$candidates" = "$expected" + +if printf '%s\n' "$candidates" | grep -Eq '(^| )c[axp]x| cx[0-9]'; then + printf 'highmem candidates must not downgrade from ccx53\n' >&2 + exit 1 +fi +if printf '%s\n' "$candidates" | grep -Evq '^ccx53 (nbg1|fsn1|hel1) amd64$'; then + printf 'highmem candidates must be ccx53 amd64 in allowed regions only\n' >&2 + exit 1 +fi + +export GCR_BUDGET_EUR_MONTHLY='6.83' +if gcr_budget_add 0.8550 480; then + printf 'highmem full-TTL reservation must obey budget cap\n' >&2 + exit 1 +fi +export GCR_BUDGET_EUR_MONTHLY='6.84' +gcr_budget_add 0.8550 480 +test "$(cat "$GCR_STATE_DIR/budget/$(date -u '+%Y-%m')")" = '6.8400' + +calls="$GCR_STATE_DIR/hcloud-calls" +sleep() { :; } +gcr_hcloud_token() { printf token; } +gcr_hcloud_req() { + method="$1"; path="$2"; body="${3:-}" + test "$method" = POST + test "$path" = /servers + printf '%s\n' "$body" | jq -c . >> "$calls" + count="$(wc -l < "$calls" | tr -d ' ')" + GCR_LAST_BODY="$GCR_STATE_DIR/last-body.json" + case "$count" in + 1|2) return 1 ;; + 3) printf '{"server":{"id":9001}}\n' > "$GCR_LAST_BODY"; return 0 ;; + *) return 1 ;; + esac +} + +vm_id="$(gcr_vm_create gcr-9001-1 gross-nix-x86-highmem ccx53 480 reg-token 9001 1 hectic-lab/util.nix)" +test "$vm_id" = '9001' +test "$(wc -l < "$calls" | tr -d ' ')" = '3' +jq -e 'select(.server_type == "ccx53" and .location == "nbg1" and .labels["gcr.arch"] == "amd64" and .labels["gcr.label"] == "gross-nix-x86-highmem" and .labels["gcr.ttl-min"] == "480")' "$calls" >/dev/null +jq -e 'select(.server_type == "ccx53" and .location == "fsn1" and .labels["gcr.arch"] == "amd64")' "$calls" >/dev/null +jq -e 'select(.server_type == "ccx53" and .location == "hel1" and .labels["gcr.arch"] == "amd64")' "$calls" >/dev/null +if jq -e 'select(.server_type != "ccx53" or .labels["gcr.arch"] != "amd64")' "$calls" >/dev/null; then + printf 'highmem VM creation attempted non-ccx53 or non-amd64 candidate\n' >&2 + exit 1 +fi diff --git a/test/package/with-attic-cache/time-budgets.py b/test/package/with-attic-cache/time-budgets.py index 165b670b..7c44dc99 100644 --- a/test/package/with-attic-cache/time-budgets.py +++ b/test/package/with-attic-cache/time-budgets.py @@ -32,10 +32,13 @@ def workflow_budget(path): with open(path, encoding="utf-8") as handle: data = yaml.safe_load(handle) job = data["jobs"]["deploy"] + workflow_on = data.get("on", data.get(True)) + inputs = workflow_on["workflow_dispatch"]["inputs"] step = next(s for s in job["steps"] if s.get("name") == "Deploy neuro") env = step["env"] return { "label": job["runs-on"], + "runner_input": inputs["runner_label"], "workflow": int(job["timeout-minutes"]), "build": int(env["WITH_ATTIC_BUILD_TIMEOUT"]) // 60, "drain": int(env["WITH_ATTIC_DRAIN_TIMEOUT"]) // 60, @@ -63,11 +66,13 @@ def shell_budgets(label): budget = workflow_budget(os.environ["WORKFLOW_FILE"]) -need(budget["label"] == "gross-nix-x86-perf", "unexpected deployment runner label") +need(budget["label"] == "${{ gitea.event.inputs.runner_label || 'nix' }}", "unexpected deployment runner expression") +need(budget["runner_input"]["default"] == "nix", "runner label default must stay nix") +need(budget["runner_input"]["options"] == ["nix", "gross-nix-x86-perf", "gross-nix-x86-highmem"], "runner label choices drifted") need(budget["build"] > 0 and budget["drain"] > 0 and budget["upload"] > 0, "non-positive timeout") need(budget["build"] + budget["drain"] + 15 <= budget["workflow"], "workflow too short for build+drain") -ttl, profile, grace, runner_timeout, other_ttls = shell_budgets(budget["label"]) +ttl, profile, grace, runner_timeout, other_ttls = shell_budgets("gross-nix-x86-perf") need(ttl == 480, f"expected 480m ttl, got {ttl}") need(profile.split()[1] == "480", f"profile ttl drifted: {profile}") need(all(line.endswith("=180") for line in other_ttls), f"default ttl drift: {other_ttls}") @@ -75,4 +80,9 @@ need(budget["workflow"] < runner_timeout, "workflow must be below runner timeout need(minutes(os.environ["GITEA_WATCHDOG"]) >= budget["workflow"], "Gitea watchdog too short") need(ttl + grace >= budget["workflow"] + 15, "VM ttl lacks bootstrap allowance") +highmem_ttl, highmem_profile, _, highmem_runner_timeout, _ = shell_budgets("gross-nix-x86-highmem") +need(highmem_ttl == 480, f"expected highmem 480m ttl, got {highmem_ttl}") +need(highmem_profile == "ccx53 480 0.8550", f"highmem profile drifted: {highmem_profile}") +need(highmem_runner_timeout == 480, f"highmem runner timeout drifted: {highmem_runner_timeout}") + print(f"PASS build={budget['build']}m drain={budget['drain']}m upload={budget['upload']}m workflow={budget['workflow']}m runner={runner_timeout}m ttl={ttl}m grace={grace}m watchdog={os.environ['GITEA_WATCHDOG']}")