fix: runners

test/package/gitea-runner-controller/test/highmem-label.sh

@@ -5,6 +5,9 @@ set -eu
 . "$STATE_SH"
 . "$DECIDE_SH"
 . "$HCLOUD_SH"
+. "$GITEA_SH"
+. "$CONTROLLER_SH"
+. "$WEBHOOK_SH"
 
 gcr_state_init
 export GCR_ALLOWED_REPOS='hectic-lab/util.nix'
@@ -13,10 +16,22 @@ export GCR_NIX_IMAGE_ID='424242'
 
 test "$(gcr_label_ttl ubuntu-latest)" = '180'
 test "$(gcr_label_ttl nix)" = '480'
-test "$(gcr_decide ubuntu-latest hectic-lab/util.nix)" = 'cx53 180 0.032'
-test "$(gcr_decide nix hectic-lab/util.nix)" = 'cx53 480 0.032'
-test "$(gcr_label_candidates ubuntu-latest | head -n1)" = 'cx53 nbg1 amd64'
-test "$(gcr_label_candidates nix | head -n1)" = 'cx53 nbg1 amd64'
+test "$(gcr_decide ubuntu-latest hectic-lab/util.nix)" = 'cx23 180 0.004'
+test "$(gcr_decide nix hectic-lab/util.nix)" = 'cx23 480 0.004'
+standard_candidates='cx23 nbg1 amd64
+cx23 fsn1 amd64
+cx23 hel1 amd64
+cx33 nbg1 amd64
+cx33 fsn1 amd64
+cx33 hel1 amd64
+cx43 nbg1 amd64
+cx43 fsn1 amd64
+cx43 hel1 amd64
+cx53 nbg1 amd64
+cx53 fsn1 amd64
+cx53 hel1 amd64'
+test "$(gcr_label_candidates ubuntu-latest)" = "$standard_candidates"
+test "$(gcr_label_candidates nix)" = "$standard_candidates"
 test "$(gcr_image_id_for_arch amd64 ubuntu-latest)" = '313131'
 test "$(gcr_image_id_for_arch amd64 nix)" = '424242'
 
@@ -44,29 +59,36 @@ if gcr_budget_can_add 0.8550 480; then
   printf 'highmem full-TTL reservation must obey budget cap\n' >&2
   exit 1
 fi
-export GCR_BUDGET_EUR_MONTHLY='6.84'
-gcr_budget_add 0.8550 480
-test "$(cat "$GCR_STATE_DIR/budget/$(date -u '+%Y-%m')")" = '6.8400'
+if gcr_budget_write "$GCR_STATE_DIR/missing/budget" 1 2>/dev/null; then
+  printf 'budget writes must propagate failures\n' >&2
+  exit 1
+fi
+rm -f "$GCR_STATE_DIR/budget/$(date -u '+%Y-%m')"
+export GCR_BUDGET_EUR_MONTHLY='10'
 
 calls="$GCR_STATE_DIR/hcloud-calls"
 sleep() { :; }
 gcr_hcloud_token() { printf token; }
 gcr_hcloud_req() {
   method="$1"; path="$2"; body="${3:-}"
-  test "$method" = POST
+  GCR_LAST_BODY="$GCR_STATE_DIR/last-body.json"
+  if [ "$method" = GET ]; then
+    printf '{"servers":[]}\n' > "$GCR_LAST_BODY"
+    GCR_LAST_HTTP=200
+    return 0
+  fi
   test "$path" = /servers
   printf '%s\n' "$body" | jq -c . >> "$calls"
   count="$(wc -l < "$calls" | tr -d ' ')"
-  GCR_LAST_BODY="$GCR_STATE_DIR/last-body.json"
   case "$count" in
-    1|2) return 1 ;;
-    3) printf '{"server":{"id":9001}}\n' > "$GCR_LAST_BODY"; return 0 ;;
+    1|2) GCR_LAST_HTTP=412; return 1 ;;
+    3) GCR_LAST_HTTP=201; printf '{"server":{"id":9001}}\n' > "$GCR_LAST_BODY"; return 0 ;;
     *) return 1 ;;
   esac
 }
 
-vm_id="$(gcr_vm_create gcr-9001-1 gross-nix-x86-highmem ccx53 480 reg-token 9001 1 hectic-lab/util.nix)"
-test "$vm_id" = '9001'
+created="$(gcr_vm_create gcr-9001-1 gross-nix-x86-highmem ccx53 480 reg-token 9001 1 hectic-lab/util.nix)"
+test "$created" = '9001 ccx53 0.8550'
 test "$(wc -l < "$calls" | tr -d ' ')" = '3'
 jq -e 'select(.server_type == "ccx53" and .location == "nbg1" and .labels["gcr.arch"] == "amd64" and .labels["gcr.label"] == "gross-nix-x86-highmem" and .labels["gcr.ttl-min"] == "480")' "$calls" >/dev/null
 jq -e 'select(.server_type == "ccx53" and .location == "fsn1" and .labels["gcr.arch"] == "amd64")' "$calls" >/dev/null
@@ -75,3 +97,210 @@ if jq -e 'select(.server_type != "ccx53" or .labels["gcr.arch"] != "amd64")' "$c
   printf 'highmem VM creation attempted non-ccx53 or non-amd64 candidate\n' >&2
   exit 1
 fi
+
+# Standard allocation charges actual fallback type after cx23 capacity failures.
+rm -f "$calls" "$GCR_STATE_DIR/budget/$(date -u '+%Y-%m')"
+export GCR_BUDGET_EUR_MONTHLY='10'
+gcr_gitea_registration_token() { printf token; }
+gcr_hcloud_req() {
+  method="$1"; path="$2"; body="${3:-}"
+  GCR_LAST_BODY="$GCR_STATE_DIR/last-body.json"
+  if [ "$method" = GET ]; then
+    printf 'lookup\n' >> "$GCR_STATE_DIR/lookups"
+    printf '{"servers":[]}\n' > "$GCR_LAST_BODY"
+    GCR_LAST_HTTP=200
+    return 0
+  fi
+  test "$path" = /servers
+  printf '%s\n' "$body" | jq -c . >> "$calls"
+  count="$(wc -l < "$calls" | tr -d ' ')"
+  case "$count" in
+    1|2|3) GCR_LAST_HTTP=412; return 1 ;;
+    4) GCR_LAST_HTTP=201; printf '{"server":{"id":9002}}\n' > "$GCR_LAST_BODY"; return 0 ;;
+    *) return 1 ;;
+  esac
+}
+
+gcr_alloc 9002 1 hectic-lab/util.nix '["nix"]'
+test "$RESPONSE_CODE" = 202
+test "$RESPONSE_BODY" = 'allocated gcr-9002-1'
+test "$(wc -l < "$calls" | tr -d ' ')" = '4'
+test "$(wc -l < "$GCR_STATE_DIR/lookups" | tr -d ' ')" = '3'
+jq -e -s 'map(.server_type) == ["cx23", "cx23", "cx23", "cx33"]' "$calls" >/dev/null
+test "$(gcr_server_hourly_rate cx33)" = '0.008'
+test "$(cat "$GCR_STATE_DIR/budget/$(date -u '+%Y-%m')")" = '0.0640'
+
+# Ambiguous 5xx response adopts only exact deterministic managed identity.
+gcr_record_del 9002 1
+rm -f "$calls" "$GCR_STATE_DIR/budget/$(date -u '+%Y-%m')"
+gcr_hcloud_req() {
+  method="$1"; body="${3:-}"
+  GCR_LAST_BODY="$GCR_STATE_DIR/last-body.json"
+  if [ "$method" = POST ]; then
+    printf '%s\n' "$body" | jq -c . >> "$calls"
+    GCR_LAST_HTTP=000
+    return 1
+  fi
+  GCR_LAST_HTTP=200
+  printf '%s\n' '{"servers":[{"id":9010,"name":"gcr-9010-1","server_type":{"name":"cx23"},"labels":{"gitea-runner-controller":"managed","gcr.job-id":"9010","gcr.run-attempt":"1","gcr.label":"nix","gcr.location":"nbg1","gcr.arch":"amd64"}}]}' > "$GCR_LAST_BODY"
+}
+gcr_alloc 9010 1 hectic-lab/util.nix '["nix"]'
+test "$RESPONSE_BODY" = 'allocated gcr-9010-1'
+test "$(wc -l < "$calls" | tr -d ' ')" = 1
+found_rec="$(gcr_record_get 9010 1)"
+test "$(gcr_record_field "$found_rec" status)" = pending_vm
+test "$(gcr_record_field "$found_rec" vm_id)" = 9010
+test "$(cat "$GCR_STATE_DIR/budget/$(date -u '+%Y-%m')")" = '0.0320'
+
+# Ambiguous POST plus failed lookup retains reservation and blocks fallback;
+# reconciler adopts a later exact identity without deleting or refunding it.
+gcr_record_del 9010 1
+rm -f "$calls" "$GCR_STATE_DIR/budget/$(date -u '+%Y-%m')" "$GCR_STATE_DIR/lookups"
+gcr_hcloud_req() {
+  method="$1"; body="${3:-}"
+  if [ "$method" = POST ]; then
+    printf '%s\n' "$body" | jq -c . >> "$calls"
+    GCR_LAST_HTTP=500
+    return 1
+  fi
+  printf 'lookup-failed\n' >> "$GCR_STATE_DIR/lookups"
+  GCR_LAST_HTTP=503
+  return 1
+}
+gcr_alloc 9012 1 hectic-lab/util.nix '["nix"]'
+test "$RESPONSE_BODY" = 'VM creation pending recovery'
+test "$(wc -l < "$calls" | tr -d ' ')" = 1
+ambiguous_rec="$(gcr_record_get 9012 1)"
+test "$(gcr_record_field "$ambiguous_rec" status)" = create_ambiguous
+test "$(gcr_record_field "$ambiguous_rec" budget_rate)" = 0.004
+test "$(gcr_count_active)" = 1
+test "$(cat "$GCR_STATE_DIR/budget/$(date -u '+%Y-%m')")" = '0.0320'
+gcr_sweep_create_ambiguous
+test "$(gcr_record_field "$(gcr_record_get 9012 1)" status)" = create_ambiguous
+test "$(wc -l < "$calls" | tr -d ' ')" = 1
+
+gcr_hcloud_req() {
+  test "$1" = GET
+  GCR_LAST_BODY="$GCR_STATE_DIR/last-body.json"
+  GCR_LAST_HTTP=200
+  printf '%s\n' '{"servers":[{"id":9012,"name":"gcr-9012-1","server_type":{"name":"cx23"},"labels":{"gitea-runner-controller":"managed","gcr.job-id":"9012","gcr.run-attempt":"1","gcr.label":"nix","gcr.location":"nbg1","gcr.arch":"amd64"}}]}' > "$GCR_LAST_BODY"
+}
+gcr_vm_destroy() { printf 'unexpected-destroy %s\n' "$1" >> "$GCR_STATE_DIR/ambiguity-destroys"; return 1; }
+gcr_sweep_create_ambiguous
+recovered_rec="$(gcr_record_get 9012 1)"
+test "$(gcr_record_field "$recovered_rec" status)" = pending_vm
+test "$(gcr_record_field "$recovered_rec" vm_id)" = 9012
+test "$(gcr_record_field "$recovered_rec" server_type)" = cx23
+test "$(gcr_record_field "$recovered_rec" budget_rate)" = 0.004
+test "$(cat "$GCR_STATE_DIR/budget/$(date -u '+%Y-%m')")" = '0.0320'
+test ! -e "$GCR_STATE_DIR/ambiguity-destroys"
+gcr_record_del 9012 1
+rm -f "$GCR_STATE_DIR/budget/$(date -u '+%Y-%m')"
+
+# Fallback candidates exceeding remaining budget never reach Hetzner.
+rm -f "$calls" "$GCR_STATE_DIR/budget/$(date -u '+%Y-%m')"
+export GCR_BUDGET_EUR_MONTHLY='0.05'
+gcr_hcloud_req() {
+  method="$1"; path="$2"; body="${3:-}"
+  GCR_LAST_BODY="$GCR_STATE_DIR/last-body.json"
+  if [ "$method" = GET ]; then
+    printf '{"servers":[]}\n' > "$GCR_LAST_BODY"
+    GCR_LAST_HTTP=200
+    return 0
+  fi
+  test "$path" = /servers
+  printf '%s\n' "$body" | jq -c . >> "$calls"
+  GCR_LAST_HTTP=412
+  return 1
+}
+gcr_alloc 9003 1 hectic-lab/util.nix '["nix"]'
+test "$RESPONSE_BODY" = 'VM creation failed'
+test "$(wc -l < "$calls" | tr -d ' ')" = '3'
+jq -e -s 'all(.server_type == "cx23")' "$calls" >/dev/null
+test "$(cat "$GCR_STATE_DIR/budget/$(date -u '+%Y-%m')")" = '0.0000'
+
+# Deferred allocation keeps cleanup ownership and reservation across failed
+# DELETE, failed post-DELETE state update, and cleanup-owned 404 retry.
+rm -f "$calls" "$GCR_STATE_DIR/budget/$(date -u '+%Y-%m')"
+export GCR_BUDGET_EUR_MONTHLY='10'
+deferred='{"job_id":"9005","run_attempt":"1","repo":"hectic-lab/util.nix","label":"nix","created_at":"0","ttl_min":null,"vm_id":"","vm_name":"","status":"deferred"}'
+printf '%s\n' "$deferred" > "$GCR_STATE_DIR/jobs/9005-1.json"
+gcr_gitea_job_state() { printf queued; }
+record_put_count="$GCR_STATE_DIR/record-put-count"
+gcr_record_put() {
+  count="$(cat "$record_put_count" 2>/dev/null || echo 0)"
+  count=$((count + 1))
+  printf '%s\n' "$count" > "$record_put_count"
+  case "$count" in 1|3|5|8) return 1 ;; esac
+  tmp="$(mktemp "$(dirname "$(gcr_record_path "$1" "$2")")/.tmp.XXXXXX")"
+  printf '%s\n' "$3" > "$tmp"
+  mv -f "$tmp" "$(gcr_record_path "$1" "$2")"
+}
+gcr_hcloud_req() {
+  test "$1" = POST
+  printf '%s\n' "$3" | jq -c . >> "$calls"
+  GCR_LAST_BODY="$GCR_STATE_DIR/last-body.json"
+  printf '{"server":{"id":9005}}\n' > "$GCR_LAST_BODY"
+}
+gcr_vm_destroy() {
+  printf 'destroy-failed %s\n' "$1" >> "$GCR_STATE_DIR/destroy-calls"
+  return 1
+}
+gcr_alloc_deferred 9005 1
+cleanup_rec="$(gcr_record_get 9005 1)"
+test "$(gcr_record_field "$cleanup_rec" status)" = cleanup_pending
+test "$(gcr_record_field "$cleanup_rec" vm_id)" = 9005
+test "$(gcr_record_field "$cleanup_rec" budget_rate)" = 0.004
+test "$(cat "$GCR_STATE_DIR/budget/$(date -u '+%Y-%m')")" = '0.0320'
+
+gcr_vm_destroy() { printf 'deleted %s\n' "$1" >> "$GCR_STATE_DIR/destroy-calls"; }
+gcr_sweep_cleanup_pending
+cleanup_rec="$(gcr_record_get 9005 1)"
+test "$(gcr_record_field "$cleanup_rec" status)" = cleanup_pending
+test "$(gcr_record_field "$cleanup_rec" cleanup_vm_destroyed)" = false
+test "$(cat "$GCR_STATE_DIR/budget/$(date -u '+%Y-%m')")" = '0.0320'
+gcr_vm_destroy() {
+  printf 'absent %s\n' "$1" >> "$GCR_STATE_DIR/destroy-calls"
+  GCR_LAST_HTTP=404
+  return 1
+}
+gcr_sweep_cleanup_pending
+cleanup_rec="$(gcr_record_get 9005 1)"
+test "$(gcr_record_field "$cleanup_rec" cleanup_vm_destroyed)" = true
+test "$(cat "$GCR_STATE_DIR/budget/$(date -u '+%Y-%m')")" = '0.0000'
+gcr_sweep_cleanup_pending
+test ! -e "$(gcr_record_path 9005 1)"
+test "$(cat "$GCR_STATE_DIR/budget/$(date -u '+%Y-%m')")" = '0.0000'
+grep -q '^destroy-failed 9005$' "$GCR_STATE_DIR/destroy-calls"
+grep -q '^deleted 9005$' "$GCR_STATE_DIR/destroy-calls"
+grep -q '^absent 9005$' "$GCR_STATE_DIR/destroy-calls"
+test "$(grep -c '^absent 9005$' "$GCR_STATE_DIR/destroy-calls")" = 1
+
+# Confirmed ambiguous absence uses same monotonic refund claim. Failed state
+# persistence after credit cannot make retry subtract reservation twice.
+gcr_budget_add 0.004 480
+ambiguous_absent='{"job_id":"9013","run_attempt":"1","repo":"hectic-lab/util.nix","label":"nix","created_at":"0","ttl_min":480,"vm_id":"","vm_name":"gcr-9013-1","server_type":"cx23","budget_rate":"0.004","candidate_location":"nbg1","candidate_arch":"amd64","bootstrapped":false,"status":"create_ambiguous"}'
+printf '%s\n' "$ambiguous_absent" > "$GCR_STATE_DIR/jobs/9013-1.json"
+gcr_hcloud_req() {
+  test "$1" = GET
+  GCR_LAST_BODY="$GCR_STATE_DIR/last-body.json"
+  GCR_LAST_HTTP=200
+  printf '{"servers":[]}\n' > "$GCR_LAST_BODY"
+}
+gcr_sweep_create_ambiguous
+absence_rec="$(gcr_record_get 9013 1)"
+test "$(gcr_record_field "$absence_rec" status)" = cleanup_pending
+test "$(gcr_record_field "$absence_rec" cleanup_vm_destroyed)" = true
+test "$(cat "$GCR_STATE_DIR/budget/$(date -u '+%Y-%m')")" = '0.0000'
+gcr_sweep_cleanup_pending
+test ! -e "$(gcr_record_path 9013 1)"
+test "$(cat "$GCR_STATE_DIR/budget/$(date -u '+%Y-%m')")" = '0.0000'
+
+# Reservation write failure aborts before any Hetzner request.
+rm -f "$calls"
+rm -rf "$GCR_STATE_DIR/budget"
+printf 'not-a-directory\n' > "$GCR_STATE_DIR/budget"
+gcr_alloc 9006 1 hectic-lab/util.nix '["nix"]'
+test "$RESPONSE_BODY" = 'VM creation failed'
+test ! -e "$calls"
+test ! -e "$(gcr_record_path 9006 1)"

test/package/gitea-runner-controller/test/idle-reuse.sh

@@ -19,7 +19,11 @@ gcr_now_epoch() { printf '%s' "$NOW"; }
 calls="$GCR_STATE_DIR/calls"
 gcr_budget_add() { printf 'budget\n' >> "$calls"; }
 gcr_gitea_registration_token() { printf 'token'; printf 'token\n' >> "$calls"; }
-gcr_vm_create() { printf 'create\n' >> "$calls"; printf '99'; }
+gcr_vm_create() {
+  printf 'create\n' >> "$calls"
+  gcr_budget_add 0.032 180
+  printf '99 cx53 0.032'
+}
 gcr_vm_destroy() { printf 'destroy vm=%s\n' "$1" >> "$calls"; }
 gcr_vm_runner_service() { printf 'runner %s vm=%s\n' "$2" "$1" >> "$calls"; }
 gcr_gitea_runner_disabled() { printf 'runner-disabled %s %s\n' "$2" "$3" >> "$calls"; }
@@ -270,6 +274,7 @@ set -eu
 . "$LOG_SH"
 . "$STATE_SH"
 . "$DECIDE_SH"
+. "$HCLOUD_SH"
 . "$WEBHOOK_SH"
 gcr_now_epoch() { printf '2000'; }
 gcr_budget_add() { printf 'budget %s\n' "$1" >> "$GCR_STATE_DIR/admission-calls"; }
@@ -277,7 +282,8 @@ gcr_gitea_registration_token() { printf token; }
 gcr_vm_create() {
   sleep 1
   printf 'create %s\n' "$6" >> "$GCR_STATE_DIR/admission-calls"
-  printf '%s' "$6"
+  gcr_budget_add 0.032 180
+  printf '%s cx53 0.032' "$6"
 }
 gcr_vm_destroy() { :; }
 gcr_state_init

test/package/gitea-runner-controller/test/reap-diagnostics.sh

@@ -63,6 +63,29 @@ gcr_reap_finished_jobs
 test "$(gcr_record_get 101 1)" = "$idle_once"
 test ! -e "$(gcr_record_path 102 1)"
 
+# Reaper keeps cleanup ownership after DELETE failure and retries next sweep.
+record_delete_fail='{"job_id":"104","run_attempt":"1","repo":"hinterland/hearth","label":"nix","created_at":"1","ttl_min":480,"vm_id":44,"vm_name":"gcr-104-1","bootstrapped":true,"status":"vm_active"}'
+gcr_record_put 104 1 "$record_delete_fail"
+gcr_gitea_job_state() {
+  case "$2" in
+    104) printf 'completed:failure' ;;
+    *) return 1 ;;
+  esac
+}
+gcr_vm_destroy() {
+  printf 'destroy-failed vm=%s\n' "$1" >> "$calls"
+  return 1
+}
+gcr_reap_finished_jobs
+test "$(gcr_record_field "$(gcr_record_get 104 1)" status)" = cleanup_pending
+test "$(gcr_count_active)" = 1
+
+gcr_vm_destroy() { printf 'destroy-retry vm=%s\n' "$1" >> "$calls"; }
+gcr_sweep_cleanup_pending
+test ! -e "$(gcr_record_path 104 1)"
+grep -q '^destroy-failed vm=44$' "$calls"
+grep -q '^destroy-retry vm=44$' "$calls"
+
 calls_ip_fail="$GCR_STATE_DIR/calls-ip-fail"
 calls="$calls_ip_fail"
 record_ip_fail='{"job_id":"103","run_attempt":"1","repo":"hinterland/hearth","label":"gross-nix-x86-perf","created_at":"1","ttl_min":480,"vm_id":43,"vm_name":"gcr-103-1","bootstrapped":true,"status":"vm_active"}'
@@ -79,6 +102,10 @@ gcr_vm_public_ip() {
   return 1
 }
 
+gcr_vm_destroy() {
+  printf 'destroy vm=%s\n' "$1" >> "$calls"
+}
+
 gcr_reap_finished_jobs
 
 grep -q 'diag vm=43 ip= job=103 reason=completed:failure' "$calls_ip_fail"

test/package/gitea-runner-controller/test/webhook-diagnostics.sh

@@ -71,6 +71,29 @@ grep -q 'diag vm=53 ip= job=203 reason=completed:failure' "$calls_ip_fail"
 grep -q 'destroy vm=53' "$calls_ip_fail"
 test ! -e "$(gcr_record_path 203 1)"
 
+# Terminal webhook retains ownership and capacity until DELETE succeeds.
+record_delete_fail='{"job_id":"204","run_attempt":"1","repo":"hinterland/hearth","label":"nix","created_at":"1","ttl_min":480,"vm_id":54,"vm_name":"gcr-204-1","bootstrapped":true,"status":"vm_active"}'
+gcr_record_put 204 1 "$record_delete_fail"
+gcr_vm_destroy() {
+  printf 'destroy-failed vm=%s\n' "$1" >> "$calls_ip_fail"
+  return 1
+}
+gcr_deallocate 204 1 completed:failure
+cleanup_rec="$(gcr_record_get 204 1)"
+test "$(gcr_record_field "$cleanup_rec" status)" = cleanup_pending
+test "$(gcr_count_active)" = 1
+test "$(gcr_count_active_repo hinterland/hearth)" = 1
+
+gcr_vm_destroy() { printf 'destroy-retry vm=%s\n' "$1" >> "$calls_ip_fail"; }
+key="$(gcr_alloc_key 204 1)"
+gcr_lock_acquire "$key"
+gcr_vm_cleanup_pending 204 1 "$(gcr_record_get 204 1)"
+gcr_lock_release "$key"
+test ! -e "$(gcr_record_path 204 1)"
+test "$(gcr_count_active)" = 0
+grep -q '^destroy-failed vm=54$' "$calls_ip_fail"
+grep -q '^destroy-retry vm=54$' "$calls_ip_fail"
+
 # Gitea emits zero-based run_attempt values for initial workflow jobs.
 gcr_read_request() {
   gcr_hdr_event_type=workflow_job