feat: highmem

.gitea/workflows/deploy-neuro.yaml

@@ -10,6 +10,15 @@ on:  # yamllint disable-line rule:truthy
         required: false
         type: boolean
         default: true
+      runner_label:
+        description: Runner label for deploy job
+        required: false
+        type: choice
+        default: nix
+        options:
+          - nix
+          - gross-nix-x86-perf
+          - gross-nix-x86-highmem
 
 concurrency:
   group: deploy-neuro
@@ -19,7 +28,7 @@ jobs:
   deploy:
     name: deploy neuro from performance node
     if: ${{ gitea.ref == 'refs/heads/master' }}
-    runs-on: gross-nix-x86-perf
+    runs-on: ${{ gitea.event.inputs.runner_label || 'nix' }}
     timeout-minutes: 435
     env:
       NIX_CONFIG: |

infra/gitea-runners/runbook.md

@@ -567,6 +567,7 @@ The `deploy-neuro` workflow uses these nested limits:
 | Final cache drain | 1 hour (`WITH_ATTIC_DRAIN_TIMEOUT=3600`) |
 | Workflow job | 435 minutes, including 15 minutes of setup/cleanup margin |
 | `gross-nix-x86-perf` runner | 480 minutes |
+| `gross-nix-x86-highmem` runner | 480 minutes |
 | Gitea `actions.ENDLESS_TASK_TIMEOUT` | 8 hours |
 | VM hard lifetime from allocation | 480 minutes plus 10-minute controller grace |
 
@@ -575,6 +576,12 @@ and Gitea watchdog settings before dispatching the longer workflow. Already
 allocated VMs retain the TTL and runner configuration assigned when they were
 created; updating the controller does not extend a running job.
 
+`gross-nix-x86-highmem` is an explicit costly high-memory escape hatch backed
+only by Hetzner CCX53 in `nbg1`, `fsn1`, or `hel1`; it may fall back by region
+only, never to a lower-RAM server type. Current Hetzner public pricing for
+Germany/Finland CCX53 is 0.8550 EUR/hour excluding IPv4, so one 480-minute
+allocation reserves 6.84 EUR against the controller budget before VM creation.
+
 These are maximum lifetimes: terminal jobs still trigger immediate VM teardown.
 The controller's budget reservation uses the full label TTL, so a long-running
 label reserves more of the existing monthly budget. Do not raise that budget or

package/gitea-runner-controller/decide.sh

@@ -14,6 +14,7 @@ gcr_server_hourly_rate() {
         cax21) printf '0.003' ;;
         cax31) printf '0.006' ;;
         cax41) printf '0.012' ;;
+        ccx53) printf '0.8550' ;;
         cpx52) printf '0.036' ;;
         cpx62) printf '0.072' ;;
         *) return 1 ;;
@@ -29,6 +30,7 @@ gcr_label_ttl() {
         gross-nix-x86) printf '180' ;;
         gross-nix-arm) printf '180' ;;
         gross-nix-x86-perf) printf '480' ;;
+        gross-nix-x86-highmem) printf '480' ;;
         gross-nix-mixed-econ) printf '180' ;;
         *) return 1 ;;
     esac
@@ -80,6 +82,10 @@ gcr_label_candidates() {
                 'cpx62 nbg1 amd64' 'cpx62 fsn1 amd64' 'cpx62 hel1 amd64' \
                 'cpx52 nbg1 amd64' 'cpx52 fsn1 amd64' 'cpx52 hel1 amd64'
             ;;
+        gross-nix-x86-highmem)
+            printf '%s\n' \
+                'ccx53 nbg1 amd64' 'ccx53 fsn1 amd64' 'ccx53 hel1 amd64'
+            ;;
         gross-nix-mixed-econ)
             printf '%s\n' \
                 'cx53 nbg1 amd64' 'cx53 fsn1 amd64' 'cx53 hel1 amd64' \

package/gitea-runner-controller/hcloud.sh

@@ -341,7 +341,7 @@ set -eu
 nix_arch=""
 nix_sha=""
 case "$label" in
-  nix|gross-nix-x86|gross-nix-arm|gross-nix-x86-perf|gross-nix-mixed-econ)
+  nix|gross-nix-*)
   case "\$(uname -m)" in
     x86_64)
       nix_arch=x86_64-linux

test/package/gitea-runner-controller/test/highmem-label.sh

@@ -0,0 +1,67 @@
+#!/bin/dash
+set -eu
+
+. "$LOG_SH"
+. "$STATE_SH"
+. "$DECIDE_SH"
+. "$HCLOUD_SH"
+
+gcr_state_init
+export GCR_ALLOWED_REPOS='hectic-lab/util.nix'
+export GCR_NIX_IMAGE_ID='424242'
+
+profile="$(gcr_decide gross-nix-x86-highmem hectic-lab/util.nix)"
+test "$profile" = 'ccx53 480 0.8550'
+test "$(gcr_label_ttl gross-nix-x86-highmem)" = '480'
+
+candidates="$(gcr_label_candidates gross-nix-x86-highmem)"
+expected='ccx53 nbg1 amd64
+ccx53 fsn1 amd64
+ccx53 hel1 amd64'
+test "$candidates" = "$expected"
+
+if printf '%s\n' "$candidates" | grep -Eq '(^| )c[axp]x| cx[0-9]'; then
+  printf 'highmem candidates must not downgrade from ccx53\n' >&2
+  exit 1
+fi
+if printf '%s\n' "$candidates" | grep -Evq '^ccx53 (nbg1|fsn1|hel1) amd64$'; then
+  printf 'highmem candidates must be ccx53 amd64 in allowed regions only\n' >&2
+  exit 1
+fi
+
+export GCR_BUDGET_EUR_MONTHLY='6.83'
+if gcr_budget_add 0.8550 480; then
+  printf 'highmem full-TTL reservation must obey budget cap\n' >&2
+  exit 1
+fi
+export GCR_BUDGET_EUR_MONTHLY='6.84'
+gcr_budget_add 0.8550 480
+test "$(cat "$GCR_STATE_DIR/budget/$(date -u '+%Y-%m')")" = '6.8400'
+
+calls="$GCR_STATE_DIR/hcloud-calls"
+sleep() { :; }
+gcr_hcloud_token() { printf token; }
+gcr_hcloud_req() {
+  method="$1"; path="$2"; body="${3:-}"
+  test "$method" = POST
+  test "$path" = /servers
+  printf '%s\n' "$body" | jq -c . >> "$calls"
+  count="$(wc -l < "$calls" | tr -d ' ')"
+  GCR_LAST_BODY="$GCR_STATE_DIR/last-body.json"
+  case "$count" in
+    1|2) return 1 ;;
+    3) printf '{"server":{"id":9001}}\n' > "$GCR_LAST_BODY"; return 0 ;;
+    *) return 1 ;;
+  esac
+}
+
+vm_id="$(gcr_vm_create gcr-9001-1 gross-nix-x86-highmem ccx53 480 reg-token 9001 1 hectic-lab/util.nix)"
+test "$vm_id" = '9001'
+test "$(wc -l < "$calls" | tr -d ' ')" = '3'
+jq -e 'select(.server_type == "ccx53" and .location == "nbg1" and .labels["gcr.arch"] == "amd64" and .labels["gcr.label"] == "gross-nix-x86-highmem" and .labels["gcr.ttl-min"] == "480")' "$calls" >/dev/null
+jq -e 'select(.server_type == "ccx53" and .location == "fsn1" and .labels["gcr.arch"] == "amd64")' "$calls" >/dev/null
+jq -e 'select(.server_type == "ccx53" and .location == "hel1" and .labels["gcr.arch"] == "amd64")' "$calls" >/dev/null
+if jq -e 'select(.server_type != "ccx53" or .labels["gcr.arch"] != "amd64")' "$calls" >/dev/null; then
+  printf 'highmem VM creation attempted non-ccx53 or non-amd64 candidate\n' >&2
+  exit 1
+fi

test/package/with-attic-cache/time-budgets.py

@@ -32,10 +32,13 @@ def workflow_budget(path):
     with open(path, encoding="utf-8") as handle:
         data = yaml.safe_load(handle)
     job = data["jobs"]["deploy"]
+    workflow_on = data.get("on", data.get(True))
+    inputs = workflow_on["workflow_dispatch"]["inputs"]
     step = next(s for s in job["steps"] if s.get("name") == "Deploy neuro")
     env = step["env"]
     return {
         "label": job["runs-on"],
+        "runner_input": inputs["runner_label"],
         "workflow": int(job["timeout-minutes"]),
         "build": int(env["WITH_ATTIC_BUILD_TIMEOUT"]) // 60,
         "drain": int(env["WITH_ATTIC_DRAIN_TIMEOUT"]) // 60,
@@ -63,11 +66,13 @@ def shell_budgets(label):
 
 
 budget = workflow_budget(os.environ["WORKFLOW_FILE"])
-need(budget["label"] == "gross-nix-x86-perf", "unexpected deployment runner label")
+need(budget["label"] == "${{ gitea.event.inputs.runner_label || 'nix' }}", "unexpected deployment runner expression")
+need(budget["runner_input"]["default"] == "nix", "runner label default must stay nix")
+need(budget["runner_input"]["options"] == ["nix", "gross-nix-x86-perf", "gross-nix-x86-highmem"], "runner label choices drifted")
 need(budget["build"] > 0 and budget["drain"] > 0 and budget["upload"] > 0, "non-positive timeout")
 need(budget["build"] + budget["drain"] + 15 <= budget["workflow"], "workflow too short for build+drain")
 
-ttl, profile, grace, runner_timeout, other_ttls = shell_budgets(budget["label"])
+ttl, profile, grace, runner_timeout, other_ttls = shell_budgets("gross-nix-x86-perf")
 need(ttl == 480, f"expected 480m ttl, got {ttl}")
 need(profile.split()[1] == "480", f"profile ttl drifted: {profile}")
 need(all(line.endswith("=180") for line in other_ttls), f"default ttl drift: {other_ttls}")
@@ -75,4 +80,9 @@ need(budget["workflow"] < runner_timeout, "workflow must be below runner timeout
 need(minutes(os.environ["GITEA_WATCHDOG"]) >= budget["workflow"], "Gitea watchdog too short")
 need(ttl + grace >= budget["workflow"] + 15, "VM ttl lacks bootstrap allowance")
 
+highmem_ttl, highmem_profile, _, highmem_runner_timeout, _ = shell_budgets("gross-nix-x86-highmem")
+need(highmem_ttl == 480, f"expected highmem 480m ttl, got {highmem_ttl}")
+need(highmem_profile == "ccx53 480 0.8550", f"highmem profile drifted: {highmem_profile}")
+need(highmem_runner_timeout == 480, f"highmem runner timeout drifted: {highmem_runner_timeout}")
+
 print(f"PASS build={budget['build']}m drain={budget['drain']}m upload={budget['upload']}m workflow={budget['workflow']}m runner={runner_timeout}m ttl={ttl}m grace={grace}m watchdog={os.environ['GITEA_WATCHDOG']}")