diff --git a/package/gitea-runner-controller/controller.sh b/package/gitea-runner-controller/controller.sh index 06fa2fde..22dfd83a 100644 --- a/package/gitea-runner-controller/controller.sh +++ b/package/gitea-runner-controller/controller.sh @@ -40,10 +40,16 @@ gcr_sweep_ttl() { fi gcr_log info --ns=sweep "idle slot expired job=$job_id vm=$vm_id" if [ -n "$vm_id" ] && [ "$vm_id" != "null" ] && [ "$vm_id" != "0" ]; then - gcr_vm_destroy "$vm_id" || true - gcr_event "vm-destroyed" "$job_id" "{\"vm_id\":$vm_id,\"reason\":\"idle-expired\"}" + if gcr_vm_cleanup_start "$job_id" "$attempt" "$rec" idle-expired false; then + gcr_event "vm-destroyed" "$job_id" \ + "{\"vm_id\":$vm_id,\"reason\":\"idle-expired\"}" + else + gcr_event "vm-cleanup-pending" "$job_id" \ + "{\"vm_id\":$vm_id,\"reason\":\"idle-expired\"}" + fi + else + gcr_record_del "$job_id" "$attempt" fi - gcr_record_del "$job_id" "$attempt" gcr_lock_release idle-pool continue fi @@ -78,11 +84,17 @@ gcr_sweep_ttl() { if [ -n "$vm_id" ] && [ "$vm_id" != "null" ] && [ "$vm_id" != "0" ]; then ip="$(gcr_vm_public_ip "$vm_id" || true)" gcr_vm_collect_diagnostics "$vm_id" "$ip" "$job_id" ttl || true - gcr_vm_destroy "$vm_id" || true - gcr_event "vm-destroyed" "$job_id" "{\"vm_id\":$vm_id,\"reason\":\"ttl\"}" + if gcr_vm_cleanup_start "$job_id" "$attempt" "$rec" ttl false; then + gcr_event "vm-destroyed" "$job_id" \ + "{\"vm_id\":$vm_id,\"reason\":\"ttl\"}" + else + gcr_event "vm-cleanup-pending" "$job_id" \ + "{\"vm_id\":$vm_id,\"reason\":\"ttl\"}" + fi + else + gcr_record_del "$job_id" "$attempt" fi gcr_event "job-ttl-expired" "$job_id" "{\"age\":$age}" - gcr_record_del "$job_id" "$attempt" gcr_lock_release "$key" fi done @@ -108,8 +120,20 @@ gcr_sweep_orphan_vms() { if ! gcr_record_exists_for_vm_id "$vm_id"; then gcr_log warn --ns=sweep "orphan VM $vm_id job=$jid attempt=$att -> destroy" - gcr_vm_destroy "$vm_id" || true - gcr_event "orphan-vm-destroyed" "${jid:-unknown}" "{\"vm_id\":$vm_id}" + cleanup_job="${jid:-orphan-$vm_id}" + cleanup_attempt="${att:-0}" + cleanup_rec="$(jq -n --arg j "$cleanup_job" --arg a "$cleanup_attempt" \ + --arg v "$vm_id" --arg vn "$(printf '%s' "$vm" | jq -r '.name // ""')" \ + '{job_id:$j, run_attempt:$a, repo:"", label:"", created_at:"0", + ttl_min:0, vm_id:($v|tonumber), vm_name:$vn, + bootstrapped:false, status:"cleanup_pending"}')" + if gcr_vm_cleanup_start "$cleanup_job" "$cleanup_attempt" \ + "$cleanup_rec" orphan false; then + gcr_event "orphan-vm-destroyed" "$cleanup_job" "{\"vm_id\":$vm_id}" + else + gcr_event "vm-cleanup-pending" "$cleanup_job" \ + "{\"vm_id\":$vm_id,\"reason\":\"orphan\"}" + fi fi i=$((i + 1)) done @@ -149,7 +173,7 @@ gcr_alloc_deferred() { return 0 } set -- $profile - server_type="$1"; ttl_min="$2"; rate="$3" + server_type="$1"; ttl_min="$2" gcr_lock_acquire admission || { gcr_lock_release "$key" @@ -190,11 +214,6 @@ gcr_alloc_deferred() { return 0 fi - gcr_budget_can_add "$rate" "$ttl_min" || { - gcr_lock_release admission - gcr_lock_release "$key" - return 0 - } reg_token="$(gcr_gitea_registration_token "$repo")" || { gcr_lock_release admission gcr_lock_release "$key" @@ -203,23 +222,19 @@ gcr_alloc_deferred() { vm_name="gcr-${job_id}-${attempt}" created_at="$(gcr_now_epoch)" - vm_id="$(gcr_vm_create "$vm_name" "$label" "$server_type" "$ttl_min" \ - "$reg_token" "$job_id" "$attempt" "$repo")" && [ -n "$vm_id" ] || { + create_status=0 + created="$(gcr_vm_create "$vm_name" "$label" "$server_type" "$ttl_min" \ + "$reg_token" "$job_id" "$attempt" "$repo")" || create_status="$?" + if [ "$create_status" -ne 0 ] || [ -z "$created" ]; then gcr_lock_release admission gcr_lock_release "$key" return 0 - } - - gcr_budget_add "$rate" "$ttl_min" - - rec="$(jq -n --arg j "$job_id" --arg a "$attempt" --arg r "$repo" \ - --arg l "$label" --arg t "$created_at" --arg v "$vm_id" \ - --arg vn "$vm_name" --arg ttl "$ttl_min" \ - '{job_id:$j, run_attempt:$a, repo:$r, label:$l, - created_at:$t, ttl_min:($ttl|tonumber), vm_id:($v|tonumber), - vm_name:$vn, bootstrapped:false, status:"pending_vm"}')" - if ! gcr_record_put "$job_id" "$attempt" "$rec"; then - gcr_vm_destroy "$vm_id" || true + fi + set -- $created + vm_id="$1"; actual_server_type="$2"; actual_rate="$3" + if ! gcr_vm_record_created "$job_id" "$attempt" "$repo" "$label" \ + "$created_at" "$vm_id" "$vm_name" "$ttl_min" \ + "$actual_server_type" "$actual_rate"; then gcr_lock_release admission gcr_lock_release "$key" return 0 @@ -230,6 +245,83 @@ gcr_alloc_deferred() { gcr_log info --ns=alloc "deferred job=$job_id allocated vm=$vm_id" } +gcr_sweep_cleanup_pending() { + for f in $(gcr_active_records); do + rec="$(cat "$f")" + [ "$(gcr_record_field "$rec" status)" = "cleanup_pending" ] || continue + job_id="$(gcr_record_field "$rec" job_id)" + attempt="$(gcr_record_field "$rec" run_attempt)" + key="$(gcr_alloc_key "$job_id" "$attempt")" + gcr_lock_acquire "$key" || continue + if ! gcr_lock_acquire admission; then + gcr_lock_release "$key" + continue + fi + rec="$(gcr_record_get "$job_id" "$attempt")" + if [ "$(gcr_record_field "$rec" status)" = "cleanup_pending" ]; then + gcr_vm_cleanup_pending "$job_id" "$attempt" "$rec" || true + fi + gcr_lock_release admission + gcr_lock_release "$key" + done +} + +gcr_sweep_create_ambiguous() { + for f in $(gcr_active_records); do + rec="$(cat "$f")" + [ "$(gcr_record_field "$rec" status)" = "create_ambiguous" ] || continue + job_id="$(gcr_record_field "$rec" job_id)" + attempt="$(gcr_record_field "$rec" run_attempt)" + key="$(gcr_alloc_key "$job_id" "$attempt")" + gcr_lock_acquire "$key" || continue + if ! gcr_lock_acquire admission; then + gcr_lock_release "$key" + continue + fi + rec="$(gcr_record_get "$job_id" "$attempt")" + if [ "$(gcr_record_field "$rec" status)" != "create_ambiguous" ]; then + gcr_lock_release admission + gcr_lock_release "$key" + continue + fi + find_status=0 + found_vm_id="$(gcr_vm_find_created \ + "$(gcr_record_field "$rec" vm_name)" "$job_id" "$attempt" \ + "$(gcr_record_field "$rec" label)" \ + "$(gcr_record_field "$rec" server_type)" \ + "$(gcr_record_field "$rec" candidate_location)" \ + "$(gcr_record_field "$rec" candidate_arch)")" || find_status="$?" + case "$find_status" in + 0) + rec="$(printf '%s' "$rec" | jq -c --arg vm "$found_vm_id" \ + '.vm_id = ($vm | tonumber) + | .status = "pending_vm" + | .bootstrapped = false + | del(.create_http, .create_curl_status, + .candidate_location, .candidate_arch)')" + if gcr_record_put "$job_id" "$attempt" "$rec"; then + gcr_event "vm-create-recovered" "$job_id" \ + "{\"vm_id\":$found_vm_id,\"label\":\"$(gcr_record_field "$rec" label)\"}" + fi + ;; + 1) + rec="$(printf '%s' "$rec" | jq -c \ + '.status = "cleanup_pending" + | .cleanup_reason = "ambiguous-create-absent" + | .cleanup_refund_budget = true + | .cleanup_vm_destroyed = true + | .cleanup_budget_released = false')" + if gcr_record_put "$job_id" "$attempt" "$rec"; then + gcr_vm_cleanup_pending "$job_id" "$attempt" "$rec" || true + fi + ;; + 2) ;; + esac + gcr_lock_release admission + gcr_lock_release "$key" + done +} + gcr_retry_deferred() { for f in $(gcr_active_records); do rec="$(cat "$f")" @@ -343,10 +435,17 @@ gcr_bootstrap_pending() { gcr_vm_collect_diagnostics "$vm_id" "$ip" "$job_id" "$state" || true ;; esac - gcr_vm_destroy "$vm_id" || true - gcr_event "vm-destroyed" "$job_id" "{\"vm_id\":$vm_id,\"reason\":\"pending-job-completed\",\"state\":\"$state\"}" + if gcr_vm_cleanup_start "$job_id" "$attempt" "$rec" \ + pending-job-completed false; then + gcr_event "vm-destroyed" "$job_id" \ + "{\"vm_id\":$vm_id,\"reason\":\"pending-job-completed\",\"state\":\"$state\"}" + else + gcr_event "vm-cleanup-pending" "$job_id" \ + "{\"vm_id\":$vm_id,\"reason\":\"pending-job-completed\"}" + fi + else + gcr_record_del "$job_id" "$attempt" fi - gcr_record_del "$job_id" "$attempt" gcr_lock_release "$key" continue ;; @@ -418,10 +517,15 @@ gcr_reap_finished_jobs() { if ! gcr_gitea_runner_disabled "$repo" "$runner_name" true \ || ! gcr_vm_runner_service "$vm_id" stop; then gcr_lock_release idle-pool - gcr_vm_destroy "$vm_id" || true - gcr_record_del "$job_id" "$attempt" + if gcr_vm_cleanup_start "$job_id" "$attempt" "$rec" \ + idle-stop-failed false; then + gcr_event "vm-destroyed" "$job_id" \ + "{\"vm_id\":$vm_id,\"reason\":\"idle-stop-failed\",\"via\":\"reconcile\"}" + else + gcr_event "vm-cleanup-pending" "$job_id" \ + "{\"vm_id\":$vm_id,\"reason\":\"idle-stop-failed\",\"via\":\"reconcile\"}" + fi gcr_lock_release "$key" - gcr_event "vm-destroyed" "$job_id" "{\"vm_id\":$vm_id,\"reason\":\"idle-stop-failed\",\"via\":\"reconcile\"}" continue fi gcr_record_put "$job_id" "$attempt" "$idle_rec" @@ -442,10 +546,17 @@ gcr_reap_finished_jobs() { gcr_vm_collect_diagnostics "$vm_id" "$ip" "$job_id" "$state" || true ;; esac - gcr_vm_destroy "$vm_id" || true - gcr_event "vm-destroyed" "$job_id" "{\"vm_id\":$vm_id,\"reason\":\"job-completed\",\"state\":\"$state\"}" + if gcr_vm_cleanup_start "$job_id" "$attempt" "$rec" \ + job-completed false; then + gcr_event "vm-destroyed" "$job_id" \ + "{\"vm_id\":$vm_id,\"reason\":\"job-completed\",\"state\":\"$state\"}" + else + gcr_event "vm-cleanup-pending" "$job_id" \ + "{\"vm_id\":$vm_id,\"reason\":\"job-completed\",\"state\":\"$state\"}" + fi + else + gcr_record_del "$job_id" "$attempt" fi - gcr_record_del "$job_id" "$attempt" gcr_lock_release "$key" ;; esac @@ -453,6 +564,8 @@ gcr_reap_finished_jobs() { } gcr_tick() { + gcr_sweep_create_ambiguous + gcr_sweep_cleanup_pending gcr_sweep_ttl gcr_reap_finished_jobs gcr_sweep_orphan_vms diff --git a/package/gitea-runner-controller/decide.sh b/package/gitea-runner-controller/decide.sh index fb59ed5f..ba570e27 100644 --- a/package/gitea-runner-controller/decide.sh +++ b/package/gitea-runner-controller/decide.sh @@ -41,7 +41,14 @@ gcr_label_ttl() { gcr_label_candidates() { label="$1" case "$label" in - ubuntu-latest|gross-x86) + ubuntu-latest) + printf '%s\n' \ + 'cx23 nbg1 amd64' 'cx23 fsn1 amd64' 'cx23 hel1 amd64' \ + 'cx33 nbg1 amd64' 'cx33 fsn1 amd64' 'cx33 hel1 amd64' \ + 'cx43 nbg1 amd64' 'cx43 fsn1 amd64' 'cx43 hel1 amd64' \ + 'cx53 nbg1 amd64' 'cx53 fsn1 amd64' 'cx53 hel1 amd64' + ;; + gross-x86) printf '%s\n' \ 'cx53 nbg1 amd64' 'cx53 fsn1 amd64' 'cx53 hel1 amd64' \ 'cx43 nbg1 amd64' 'cx43 fsn1 amd64' 'cx43 hel1 amd64' \ @@ -65,7 +72,14 @@ gcr_label_candidates() { 'cax41 nbg1 arm64' 'cax41 fsn1 arm64' 'cax41 hel1 arm64' \ 'cx43 nbg1 amd64' 'cx43 fsn1 amd64' 'cx43 hel1 amd64' ;; - nix|gross-nix-x86) + nix) + printf '%s\n' \ + 'cx23 nbg1 amd64' 'cx23 fsn1 amd64' 'cx23 hel1 amd64' \ + 'cx33 nbg1 amd64' 'cx33 fsn1 amd64' 'cx33 hel1 amd64' \ + 'cx43 nbg1 amd64' 'cx43 fsn1 amd64' 'cx43 hel1 amd64' \ + 'cx53 nbg1 amd64' 'cx53 fsn1 amd64' 'cx53 hel1 amd64' + ;; + gross-nix-x86) printf '%s\n' \ 'cx53 nbg1 amd64' 'cx53 fsn1 amd64' 'cx53 hel1 amd64' \ 'cx43 nbg1 amd64' 'cx43 fsn1 amd64' 'cx43 hel1 amd64' \ @@ -148,7 +162,7 @@ gcr_count_active() { for f in $(gcr_active_records); do status="$(gcr_record_field "$(cat "$f")" status)" case "$status" in - pending_vm|vm_active) count=$((count + 1)) ;; + pending_vm|vm_active|cleanup_pending|create_ambiguous) count=$((count + 1)) ;; esac done printf '%s' "$count" @@ -160,7 +174,7 @@ gcr_count_active_repo() { for f in $(gcr_active_records); do rec="$(cat "$f")" case "$(gcr_record_field "$rec" status)" in - pending_vm|vm_active) ;; + pending_vm|vm_active|cleanup_pending|create_ambiguous) ;; *) continue ;; esac [ "$(gcr_record_field "$rec" repo)" = "$repo" ] && count=$((count + 1)) diff --git a/package/gitea-runner-controller/hcloud.sh b/package/gitea-runner-controller/hcloud.sh index a088aee4..0a6d76c5 100644 --- a/package/gitea-runner-controller/hcloud.sh +++ b/package/gitea-runner-controller/hcloud.sh @@ -36,6 +36,7 @@ gcr_hcloud_req() { method="$1"; path="$2"; body="${3:-}" token="$(gcr_hcloud_token)" || return 1 GCR_LAST_BODY="$(mktemp "${TMPDIR:-/tmp}/gcr-resp.XXXXXX")" + curl_status=0 if [ -n "$body" ]; then code="$(printf '%s' "$body" | curl -sS -X "$method" \ -H "Authorization: Bearer $token" \ @@ -43,13 +44,20 @@ gcr_hcloud_req() { --data-binary @- \ -o "$GCR_LAST_BODY" \ -w '%{http_code}' \ - "$GCR_API$path")" + "$GCR_API$path")" || curl_status="$?" else code="$(curl -sS -X "$method" \ -H "Authorization: Bearer $token" \ -o "$GCR_LAST_BODY" \ -w '%{http_code}' \ - "$GCR_API$path")" + "$GCR_API$path")" || curl_status="$?" + fi + case "$code" in ''|*[!0-9]*) code=000 ;; esac + GCR_LAST_CURL_STATUS="$curl_status" + GCR_LAST_HTTP="$code" + if [ "$curl_status" -ne 0 ]; then + gcr_log warn --ns=hcloud "transport failed path=$path curl=$curl_status http=$code" + return 1 fi case "$code" in 2??) return 0 ;; esac gcr_log warn --ns=hcloud "request failed path=$path http=$code body=$(head -c 200 "$GCR_LAST_BODY" | gcr_redact)" @@ -62,6 +70,64 @@ gcr_vm_list_managed() { fi } +# Exact deterministic create identity. Exit 0 = one match (prints id), +# 1 = confirmed absent, 2 = lookup failed or identity invariant violated. +gcr_vm_find_created() { + find_name="$1"; find_job="$2"; find_attempt="$3"; find_label="$4" + find_type="$5"; find_location="$6"; find_arch="$7" + if ! gcr_hcloud_req GET "/servers?name=$find_name"; then + return 2 + fi + find_matches="$(jq -c \ + --arg name "$find_name" --arg job "$find_job" --arg attempt "$find_attempt" \ + --arg label "$find_label" --arg type "$find_type" \ + --arg location "$find_location" --arg arch "$find_arch" \ + '[.servers[] | select( + .name == $name + and .labels["gitea-runner-controller"] == "managed" + and .labels["gcr.job-id"] == $job + and .labels["gcr.run-attempt"] == $attempt + and .labels["gcr.label"] == $label + and .labels["gcr.location"] == $location + and .labels["gcr.arch"] == $arch + and ((.server_type.name // .server_type) == $type))]' \ + "$GCR_LAST_BODY")" || return 2 + find_count="$(printf '%s' "$find_matches" | jq 'length')" || return 2 + case "$find_count" in + 0) return 1 ;; + 1) printf '%s' "$find_matches" | jq -r '.[0].id' ;; + *) return 2 ;; + esac +} + +gcr_create_explicitly_rejected() { + case "$1" in + 400|401|403|404|405|409|412|422|423) return 0 ;; + *) return 1 ;; + esac +} + +gcr_vm_record_create_ambiguous() { + ambiguous_name="$1"; ambiguous_label="$2"; ambiguous_ttl="$3" + ambiguous_job="$4"; ambiguous_attempt="$5"; ambiguous_repo="$6" + ambiguous_type="$7"; ambiguous_rate="$8"; ambiguous_location="$9" + shift 9; ambiguous_arch="$1"; ambiguous_http="$2"; ambiguous_curl="$3" + ambiguous_rec="$(jq -n --arg j "$ambiguous_job" --arg a "$ambiguous_attempt" \ + --arg r "$ambiguous_repo" --arg l "$ambiguous_label" \ + --arg t "$(gcr_now_epoch)" --arg vn "$ambiguous_name" \ + --arg ttl "$ambiguous_ttl" --arg st "$ambiguous_type" \ + --arg rate "$ambiguous_rate" --arg loc "$ambiguous_location" \ + --arg arch "$ambiguous_arch" --arg http "$ambiguous_http" \ + --arg curl "$ambiguous_curl" \ + '{job_id:$j, run_attempt:$a, repo:$r, label:$l, + created_at:$t, ttl_min:($ttl|tonumber), vm_id:"", vm_name:$vn, + server_type:$st, budget_rate:$rate, candidate_location:$loc, + candidate_arch:$arch, create_http:$http, create_curl_status:$curl, + bootstrapped:false, + status:"create_ambiguous"}')" + gcr_record_put "$ambiguous_job" "$ambiguous_attempt" "$ambiguous_rec" +} + gcr_vm_build_userdata() { vm_name="$1"; label="$2"; reg_token="$3" @@ -163,9 +229,10 @@ runcmd: } # gcr_vm_create NAME LABEL SERVER_TYPE TTL_MIN REG_TOKEN JOB_ID ATTEMPT REPO -# Prints new server id. +# Caller holds admission lock. Reserves each affordable candidate before its +# create request; prints new server id, actual server type, and reserved rate. gcr_vm_create() { - vm_name="$1"; label="$2"; server_type="$3"; ttl_min="$4" + vm_name="$1"; label="$2"; ttl_min="$4" reg_token="$5"; job_id="$6"; attempt="$7"; repo="$8" ttl_min="$(gcr_label_ttl "$label")" || return 1 @@ -174,8 +241,18 @@ gcr_vm_create() { while read -r candidate_type candidate_loc candidate_arch; do [ -n "${candidate_type:-}" ] || continue candidate_n=$((candidate_n + 1)) + candidate_rate="$(gcr_server_hourly_rate "$candidate_type")" || continue + if ! gcr_budget_can_add "$candidate_rate" "$ttl_min"; then + gcr_log info --ns=hcloud "skip candidate[$candidate_n] label=$label type=$candidate_type over budget" + continue + fi + if ! gcr_budget_add "$candidate_rate" "$ttl_min"; then + gcr_log error --ns=hcloud "budget reservation write failed label=$label type=$candidate_type" + return 1 + fi image_id="$(gcr_image_id_for_arch "$candidate_arch" "$label")" || { gcr_log warn --ns=hcloud "skip candidate[$candidate_n] label=$label arch=$candidate_arch no image" + gcr_budget_sub "$candidate_rate" "$ttl_min" || return 1 continue } payload="$(jq -n \ @@ -203,9 +280,34 @@ gcr_vm_create() { "gcr.created-at":$ts, "gcr.ttl-min":$ttl}}')" gcr_log info --ns=hcloud "try candidate[$candidate_n] label=$label type=$candidate_type arch=$candidate_arch loc=$candidate_loc" if gcr_hcloud_req POST /servers "$payload"; then - jq -r '.server.id' "$GCR_LAST_BODY" + printf '%s %s %s\n' \ + "$(jq -r '.server.id' "$GCR_LAST_BODY")" "$candidate_type" "$candidate_rate" return 0 fi + create_http="${GCR_LAST_HTTP:-000}" + create_curl="${GCR_LAST_CURL_STATUS:-0}" + find_status=0 + found_vm_id="$(gcr_vm_find_created "$vm_name" "$job_id" "$attempt" \ + "$label" "$candidate_type" "$candidate_loc" "$candidate_arch")" \ + || find_status="$?" + if [ "$find_status" -eq 0 ]; then + printf '%s %s %s\n' "$found_vm_id" "$candidate_type" "$candidate_rate" + return 0 + fi + if [ "$find_status" -eq 1 ] \ + && gcr_create_explicitly_rejected "$create_http"; then + if ! gcr_budget_sub "$candidate_rate" "$ttl_min"; then + gcr_log error --ns=hcloud "budget reservation rollback failed label=$label type=$candidate_type" + return 1 + fi + else + if ! gcr_vm_record_create_ambiguous "$vm_name" "$label" "$ttl_min" \ + "$job_id" "$attempt" "$repo" "$candidate_type" "$candidate_rate" \ + "$candidate_loc" "$candidate_arch" "$create_http" "$create_curl"; then + gcr_log error --ns=hcloud "cannot persist ambiguous create job=$job_id type=$candidate_type" + fi + return 2 + fi if [ "$candidate_n" -le 3 ]; then sleep 5 else @@ -217,7 +319,7 @@ EOF return 1 } -# gcr_vm_destroy SERVER_ID — idempotent best-effort destroy. +# gcr_vm_destroy SERVER_ID — success means DELETE returned HTTP 2xx. gcr_vm_destroy() { if ! gcr_hcloud_req DELETE "/servers/$1"; then gcr_log warn --ns=hcloud "destroy failed or already gone: server $1" @@ -225,6 +327,77 @@ gcr_vm_destroy() { fi } +# Only cleanup records establish prior ownership, making DELETE 404 a +# confirmed-absent success rather than an ambiguous lookup failure. +gcr_vm_destroy_owned() { + gcr_vm_destroy "$1" && return 0 + [ "${GCR_LAST_HTTP:-}" = "404" ] +} + +# Caller holds the lifecycle path's existing ownership locks. +gcr_vm_cleanup_pending() { + cleanup_job="$1"; cleanup_attempt="$2"; cleanup_rec="$3" + cleanup_vm_id="$(gcr_record_field "$cleanup_rec" vm_id)" + if [ "$(gcr_record_field "$cleanup_rec" cleanup_vm_destroyed)" != "true" ]; then + gcr_vm_destroy_owned "$cleanup_vm_id" || return 1 + cleanup_rec="$(printf '%s' "$cleanup_rec" | jq -c '.cleanup_vm_destroyed = true')" + gcr_record_put "$cleanup_job" "$cleanup_attempt" "$cleanup_rec" || return 1 + fi + if [ "$(gcr_record_field "$cleanup_rec" cleanup_refund_budget)" = "true" ] \ + && [ "$(gcr_record_field "$cleanup_rec" cleanup_budget_released)" != "true" ]; then + cleanup_rate="$(gcr_record_field "$cleanup_rec" budget_rate)" + cleanup_ttl="$(gcr_record_field "$cleanup_rec" ttl_min)" + cleanup_refund_key="$(gcr_alloc_key "$cleanup_job" "$cleanup_attempt")" + gcr_budget_refund_once "$cleanup_refund_key" "$cleanup_rate" "$cleanup_ttl" \ + || return 1 + cleanup_rec="$(printf '%s' "$cleanup_rec" | jq -c '.cleanup_budget_released = true')" + gcr_record_put "$cleanup_job" "$cleanup_attempt" "$cleanup_rec" || return 1 + fi + gcr_record_del "$cleanup_job" "$cleanup_attempt" +} + +# Persist intent before DELETE. Normal lifecycle teardown never changes budget; +# failed creation passes REFUND_BUDGET=true to release its unused reservation. +gcr_vm_cleanup_start() { + cleanup_job="$1"; cleanup_attempt="$2"; cleanup_source="$3" + cleanup_reason="$4"; cleanup_refund="$5" + cleanup_rec="$(printf '%s' "$cleanup_source" | jq -c \ + --arg reason "$cleanup_reason" --argjson refund "$cleanup_refund" \ + '.status = "cleanup_pending" + | .cleanup_reason = $reason + | .cleanup_refund_budget = $refund + | .cleanup_vm_destroyed = false + | .cleanup_budget_released = false')" + gcr_record_put "$cleanup_job" "$cleanup_attempt" "$cleanup_rec" || return 1 + gcr_vm_cleanup_pending "$cleanup_job" "$cleanup_attempt" "$cleanup_rec" +} + +# Caller holds allocation and admission locks. A failed primary write first +# persists cleanup ownership; reservation is released only after destroy. +gcr_vm_record_created() { + record_job="$1"; record_attempt="$2"; record_repo="$3"; record_label="$4" + record_created="$5"; record_vm_id="$6"; record_vm_name="$7" + record_ttl="$8"; record_type="$9"; shift 9; record_rate="$1" + record_rec="$(jq -n --arg j "$record_job" --arg a "$record_attempt" \ + --arg r "$record_repo" --arg l "$record_label" --arg t "$record_created" \ + --arg v "$record_vm_id" --arg vn "$record_vm_name" --arg ttl "$record_ttl" \ + --arg st "$record_type" --arg rate "$record_rate" \ + '{job_id:$j, run_attempt:$a, repo:$r, label:$l, + created_at:$t, ttl_min:($ttl|tonumber), vm_id:($v|tonumber), + vm_name:$vn, server_type:$st, budget_rate:$rate, + bootstrapped:false, status:"pending_vm"}')" + gcr_record_put "$record_job" "$record_attempt" "$record_rec" && return 0 + + if ! gcr_vm_cleanup_start "$record_job" "$record_attempt" "$record_rec" \ + state-write-failed true; then + cleanup_rec="$(gcr_record_get "$record_job" "$record_attempt")" + [ "$(gcr_record_field "$cleanup_rec" status)" = "cleanup_pending" ] && return 1 + gcr_log error --ns=alloc "cannot persist cleanup record job=$record_job vm=$record_vm_id" + return 1 + fi + return 1 +} + gcr_vm_public_ip() { # gcr_vm_public_ip SERVER_ID -> ipv4 or empty if gcr_hcloud_req GET "/servers/$1"; then diff --git a/package/gitea-runner-controller/state.sh b/package/gitea-runner-controller/state.sh index a01896e6..65d82c44 100644 --- a/package/gitea-runner-controller/state.sh +++ b/package/gitea-runner-controller/state.sh @@ -257,6 +257,21 @@ gcr_budget_can_add() { return 0 } +# Atomic replacement preserves last valid total when a write fails. +gcr_budget_write() { + gcr_budget_file="$1"; gcr_budget_value="$2" + gcr_budget_tmp="$(mktemp "$(dirname "$gcr_budget_file")/.budget.XXXXXX")" \ + || return 1 + if ! printf '%s\n' "$gcr_budget_value" > "$gcr_budget_tmp"; then + rm -f "$gcr_budget_tmp" + return 1 + fi + if ! mv -f "$gcr_budget_tmp" "$gcr_budget_file"; then + rm -f "$gcr_budget_tmp" + return 1 + fi +} + # Caller holds admission lock and has already checked gcr_budget_can_add. gcr_budget_add() { rate="$1"; ttl_min="$2" @@ -264,10 +279,46 @@ gcr_budget_add() { file="$GCR_STATE_DIR/budget/$month" current="$(cat "$file" 2>/dev/null || echo 0)" projected="$(awk -v c="$current" -v r="$rate" -v t="$ttl_min" 'BEGIN {printf "%.4f", c + r * t / 60}')" - printf '%s\n' "$projected" > "$file" - return 0 + gcr_budget_write "$file" "$projected" +} + +# Caller holds admission lock and is rolling back a matching budget addition. +gcr_budget_sub() { + rate="$1"; ttl_min="$2" + month="$(date -u '+%Y-%m')" + file="$GCR_STATE_DIR/budget/$month" + current="$(cat "$file" 2>/dev/null || echo 0)" + projected="$(awk -v c="$current" -v r="$rate" -v t="$ttl_min" \ + 'BEGIN {v = c - r * t / 60; if (v < 0) v = 0; printf "%.4f", v}')" + gcr_budget_write "$file" "$projected" +} + +# Claim is durable before credit. Existing claim means credit is consumed: +# it may have completed, or it may have leaked fail-closed after a crash. +# Never subtract twice when outcome between aggregate and state writes is unknown. +gcr_budget_refund_once() { + refund_key="$1"; refund_rate="$2"; refund_ttl="$3" + refund_month="$(date -u '+%Y-%m')" + refund_root="$GCR_STATE_DIR/budget/refunds/$refund_month" + refund_claim="$refund_root/$refund_key" + mkdir -p "$refund_root" || return 1 + if ! mkdir "$refund_claim" 2>/dev/null; then + if [ -d "$refund_claim" ]; then + if [ ! -f "$refund_claim/status" ]; then + gcr_log error --ns=budget \ + "refund outcome uncertain key=$refund_key; retaining fail-closed claim" + fi + return 0 + fi + return 1 + fi + if ! printf '%s %s\n' "$refund_rate" "$refund_ttl" > "$refund_claim/intent"; then + return 1 + fi + gcr_budget_sub "$refund_rate" "$refund_ttl" || return 1 + printf 'refunded\n' > "$refund_claim/status" || return 1 } gcr_active_records() { - grep -El '"status"[[:space:]]*:[[:space:]]*"(pending_vm|vm_active|idle_vm|deferred)"' "$GCR_STATE_DIR"/jobs/*.json 2>/dev/null || true + grep -El '"status"[[:space:]]*:[[:space:]]*"(pending_vm|vm_active|idle_vm|deferred|cleanup_pending|create_ambiguous)"' "$GCR_STATE_DIR"/jobs/*.json 2>/dev/null || true } diff --git a/package/gitea-runner-controller/webhook.sh b/package/gitea-runner-controller/webhook.sh index 46a586d0..526fcd8c 100644 --- a/package/gitea-runner-controller/webhook.sh +++ b/package/gitea-runner-controller/webhook.sh @@ -116,7 +116,7 @@ gcr_alloc() { fi set -- $profile - server_type="$1"; ttl_min="$2"; rate="$3" + server_type="$1"; ttl_min="$2" if ! gcr_lock_acquire admission; then rec="$(jq -n --arg j "$job_id" --arg a "$attempt" --arg r "$repo" \ @@ -183,15 +183,6 @@ gcr_alloc() { return 0 fi - if ! gcr_budget_can_add "$rate" "$ttl_min"; then - gcr_record_del "$job_id" "$attempt" - gcr_lock_release admission - gcr_lock_release "$key" - gcr_event "budget-refused" "$job_id" "{\"rate\":$rate,\"ttl_min\":$ttl_min}" - RESPONSE_CODE=202; RESPONSE_BODY="refused: monthly budget exhausted" - return 0 - fi - reg_token="$(gcr_gitea_registration_token "$repo")" || { gcr_record_del "$job_id" "$attempt" gcr_lock_release admission @@ -203,26 +194,27 @@ gcr_alloc() { vm_name="gcr-${job_id}-${attempt}" created_at="$(gcr_now_epoch)" - vm_id="$(gcr_vm_create "$vm_name" "$label" "$server_type" "$ttl_min" \ - "$reg_token" "$job_id" "$attempt" "$repo")" || { - gcr_record_del "$job_id" "$attempt" + create_status=0 + created="$(gcr_vm_create "$vm_name" "$label" "$server_type" "$ttl_min" \ + "$reg_token" "$job_id" "$attempt" "$repo")" || create_status="$?" + if [ "$create_status" -ne 0 ]; then + if [ "$create_status" -eq 2 ]; then + gcr_event "vm-create-ambiguous" "$job_id" "{}" + RESPONSE_CODE=202; RESPONSE_BODY="VM creation pending recovery" + else + gcr_record_del "$job_id" "$attempt" + gcr_event "vm-create-failed" "$job_id" "{}" + RESPONSE_CODE=202; RESPONSE_BODY="VM creation failed" + fi gcr_lock_release admission gcr_lock_release "$key" - gcr_event "vm-create-failed" "$job_id" "{}" - RESPONSE_CODE=202; RESPONSE_BODY="VM creation failed" return 0 - } - - gcr_budget_add "$rate" "$ttl_min" - - rec="$(jq -n --arg j "$job_id" --arg a "$attempt" --arg r "$repo" \ - --arg l "$label" --arg t "$created_at" --arg v "$vm_id" \ - --arg vn "$vm_name" --arg ttl "$ttl_min" \ - '{job_id:$j, run_attempt:$a, repo:$r, label:$l, - created_at:$t, ttl_min:($ttl|tonumber), vm_id:($v|tonumber), - vm_name:$vn, bootstrapped:false, status:"pending_vm"}')" - if ! gcr_record_put "$job_id" "$attempt" "$rec"; then - gcr_vm_destroy "$vm_id" || true + fi + set -- $created + vm_id="$1"; actual_server_type="$2"; actual_rate="$3" + if ! gcr_vm_record_created "$job_id" "$attempt" "$repo" "$label" \ + "$created_at" "$vm_id" "$vm_name" "$ttl_min" \ + "$actual_server_type" "$actual_rate"; then gcr_lock_release admission gcr_lock_release "$key" RESPONSE_CODE=202; RESPONSE_BODY="VM state write failed" @@ -266,10 +258,14 @@ gcr_deallocate() { if ! gcr_gitea_runner_disabled "$(gcr_record_field "$rec" repo)" "$runner_name" true \ || ! gcr_vm_runner_service "$vm_id" stop; then gcr_lock_release idle-pool - gcr_vm_destroy "$vm_id" || true - gcr_record_del "$job_id" "$attempt" + if gcr_vm_cleanup_start "$job_id" "$attempt" "$rec" idle-stop-failed false; then + gcr_event "vm-destroyed" "$job_id" \ + "{\"vm_id\":$vm_id,\"reason\":\"idle-stop-failed\"}" + else + gcr_event "vm-cleanup-pending" "$job_id" \ + "{\"vm_id\":$vm_id,\"reason\":\"idle-stop-failed\"}" + fi gcr_lock_release "$key" - gcr_event "vm-destroyed" "$job_id" "{\"vm_id\":$vm_id,\"reason\":\"idle-stop-failed\"}" return 0 fi gcr_record_put "$job_id" "$attempt" "$idle_rec" @@ -288,11 +284,16 @@ gcr_deallocate() { gcr_vm_collect_diagnostics "$vm_id" "$ip" "$job_id" "$new_status" || true ;; esac - gcr_vm_destroy "$vm_id" || true - gcr_event "vm-destroyed" "$job_id" "{\"vm_id\":$vm_id,\"reason\":\"$new_status\"}" + if gcr_vm_cleanup_start "$job_id" "$attempt" "$rec" "$new_status" false; then + gcr_event "vm-destroyed" "$job_id" \ + "{\"vm_id\":$vm_id,\"reason\":\"$new_status\"}" + else + gcr_event "vm-cleanup-pending" "$job_id" \ + "{\"vm_id\":$vm_id,\"reason\":\"$new_status\"}" + fi + else + gcr_record_del "$job_id" "$attempt" fi - - gcr_record_del "$job_id" "$attempt" gcr_lock_release "$key" } diff --git a/test/package/gitea-runner-controller/test/highmem-label.sh b/test/package/gitea-runner-controller/test/highmem-label.sh index e10e9a60..ff38a11c 100644 --- a/test/package/gitea-runner-controller/test/highmem-label.sh +++ b/test/package/gitea-runner-controller/test/highmem-label.sh @@ -5,6 +5,9 @@ set -eu . "$STATE_SH" . "$DECIDE_SH" . "$HCLOUD_SH" +. "$GITEA_SH" +. "$CONTROLLER_SH" +. "$WEBHOOK_SH" gcr_state_init export GCR_ALLOWED_REPOS='hectic-lab/util.nix' @@ -13,10 +16,22 @@ export GCR_NIX_IMAGE_ID='424242' test "$(gcr_label_ttl ubuntu-latest)" = '180' test "$(gcr_label_ttl nix)" = '480' -test "$(gcr_decide ubuntu-latest hectic-lab/util.nix)" = 'cx53 180 0.032' -test "$(gcr_decide nix hectic-lab/util.nix)" = 'cx53 480 0.032' -test "$(gcr_label_candidates ubuntu-latest | head -n1)" = 'cx53 nbg1 amd64' -test "$(gcr_label_candidates nix | head -n1)" = 'cx53 nbg1 amd64' +test "$(gcr_decide ubuntu-latest hectic-lab/util.nix)" = 'cx23 180 0.004' +test "$(gcr_decide nix hectic-lab/util.nix)" = 'cx23 480 0.004' +standard_candidates='cx23 nbg1 amd64 +cx23 fsn1 amd64 +cx23 hel1 amd64 +cx33 nbg1 amd64 +cx33 fsn1 amd64 +cx33 hel1 amd64 +cx43 nbg1 amd64 +cx43 fsn1 amd64 +cx43 hel1 amd64 +cx53 nbg1 amd64 +cx53 fsn1 amd64 +cx53 hel1 amd64' +test "$(gcr_label_candidates ubuntu-latest)" = "$standard_candidates" +test "$(gcr_label_candidates nix)" = "$standard_candidates" test "$(gcr_image_id_for_arch amd64 ubuntu-latest)" = '313131' test "$(gcr_image_id_for_arch amd64 nix)" = '424242' @@ -44,29 +59,36 @@ if gcr_budget_can_add 0.8550 480; then printf 'highmem full-TTL reservation must obey budget cap\n' >&2 exit 1 fi -export GCR_BUDGET_EUR_MONTHLY='6.84' -gcr_budget_add 0.8550 480 -test "$(cat "$GCR_STATE_DIR/budget/$(date -u '+%Y-%m')")" = '6.8400' +if gcr_budget_write "$GCR_STATE_DIR/missing/budget" 1 2>/dev/null; then + printf 'budget writes must propagate failures\n' >&2 + exit 1 +fi +rm -f "$GCR_STATE_DIR/budget/$(date -u '+%Y-%m')" +export GCR_BUDGET_EUR_MONTHLY='10' calls="$GCR_STATE_DIR/hcloud-calls" sleep() { :; } gcr_hcloud_token() { printf token; } gcr_hcloud_req() { method="$1"; path="$2"; body="${3:-}" - test "$method" = POST + GCR_LAST_BODY="$GCR_STATE_DIR/last-body.json" + if [ "$method" = GET ]; then + printf '{"servers":[]}\n' > "$GCR_LAST_BODY" + GCR_LAST_HTTP=200 + return 0 + fi test "$path" = /servers printf '%s\n' "$body" | jq -c . >> "$calls" count="$(wc -l < "$calls" | tr -d ' ')" - GCR_LAST_BODY="$GCR_STATE_DIR/last-body.json" case "$count" in - 1|2) return 1 ;; - 3) printf '{"server":{"id":9001}}\n' > "$GCR_LAST_BODY"; return 0 ;; + 1|2) GCR_LAST_HTTP=412; return 1 ;; + 3) GCR_LAST_HTTP=201; printf '{"server":{"id":9001}}\n' > "$GCR_LAST_BODY"; return 0 ;; *) return 1 ;; esac } -vm_id="$(gcr_vm_create gcr-9001-1 gross-nix-x86-highmem ccx53 480 reg-token 9001 1 hectic-lab/util.nix)" -test "$vm_id" = '9001' +created="$(gcr_vm_create gcr-9001-1 gross-nix-x86-highmem ccx53 480 reg-token 9001 1 hectic-lab/util.nix)" +test "$created" = '9001 ccx53 0.8550' test "$(wc -l < "$calls" | tr -d ' ')" = '3' jq -e 'select(.server_type == "ccx53" and .location == "nbg1" and .labels["gcr.arch"] == "amd64" and .labels["gcr.label"] == "gross-nix-x86-highmem" and .labels["gcr.ttl-min"] == "480")' "$calls" >/dev/null jq -e 'select(.server_type == "ccx53" and .location == "fsn1" and .labels["gcr.arch"] == "amd64")' "$calls" >/dev/null @@ -75,3 +97,210 @@ if jq -e 'select(.server_type != "ccx53" or .labels["gcr.arch"] != "amd64")' "$c printf 'highmem VM creation attempted non-ccx53 or non-amd64 candidate\n' >&2 exit 1 fi + +# Standard allocation charges actual fallback type after cx23 capacity failures. +rm -f "$calls" "$GCR_STATE_DIR/budget/$(date -u '+%Y-%m')" +export GCR_BUDGET_EUR_MONTHLY='10' +gcr_gitea_registration_token() { printf token; } +gcr_hcloud_req() { + method="$1"; path="$2"; body="${3:-}" + GCR_LAST_BODY="$GCR_STATE_DIR/last-body.json" + if [ "$method" = GET ]; then + printf 'lookup\n' >> "$GCR_STATE_DIR/lookups" + printf '{"servers":[]}\n' > "$GCR_LAST_BODY" + GCR_LAST_HTTP=200 + return 0 + fi + test "$path" = /servers + printf '%s\n' "$body" | jq -c . >> "$calls" + count="$(wc -l < "$calls" | tr -d ' ')" + case "$count" in + 1|2|3) GCR_LAST_HTTP=412; return 1 ;; + 4) GCR_LAST_HTTP=201; printf '{"server":{"id":9002}}\n' > "$GCR_LAST_BODY"; return 0 ;; + *) return 1 ;; + esac +} + +gcr_alloc 9002 1 hectic-lab/util.nix '["nix"]' +test "$RESPONSE_CODE" = 202 +test "$RESPONSE_BODY" = 'allocated gcr-9002-1' +test "$(wc -l < "$calls" | tr -d ' ')" = '4' +test "$(wc -l < "$GCR_STATE_DIR/lookups" | tr -d ' ')" = '3' +jq -e -s 'map(.server_type) == ["cx23", "cx23", "cx23", "cx33"]' "$calls" >/dev/null +test "$(gcr_server_hourly_rate cx33)" = '0.008' +test "$(cat "$GCR_STATE_DIR/budget/$(date -u '+%Y-%m')")" = '0.0640' + +# Ambiguous 5xx response adopts only exact deterministic managed identity. +gcr_record_del 9002 1 +rm -f "$calls" "$GCR_STATE_DIR/budget/$(date -u '+%Y-%m')" +gcr_hcloud_req() { + method="$1"; body="${3:-}" + GCR_LAST_BODY="$GCR_STATE_DIR/last-body.json" + if [ "$method" = POST ]; then + printf '%s\n' "$body" | jq -c . >> "$calls" + GCR_LAST_HTTP=000 + return 1 + fi + GCR_LAST_HTTP=200 + printf '%s\n' '{"servers":[{"id":9010,"name":"gcr-9010-1","server_type":{"name":"cx23"},"labels":{"gitea-runner-controller":"managed","gcr.job-id":"9010","gcr.run-attempt":"1","gcr.label":"nix","gcr.location":"nbg1","gcr.arch":"amd64"}}]}' > "$GCR_LAST_BODY" +} +gcr_alloc 9010 1 hectic-lab/util.nix '["nix"]' +test "$RESPONSE_BODY" = 'allocated gcr-9010-1' +test "$(wc -l < "$calls" | tr -d ' ')" = 1 +found_rec="$(gcr_record_get 9010 1)" +test "$(gcr_record_field "$found_rec" status)" = pending_vm +test "$(gcr_record_field "$found_rec" vm_id)" = 9010 +test "$(cat "$GCR_STATE_DIR/budget/$(date -u '+%Y-%m')")" = '0.0320' + +# Ambiguous POST plus failed lookup retains reservation and blocks fallback; +# reconciler adopts a later exact identity without deleting or refunding it. +gcr_record_del 9010 1 +rm -f "$calls" "$GCR_STATE_DIR/budget/$(date -u '+%Y-%m')" "$GCR_STATE_DIR/lookups" +gcr_hcloud_req() { + method="$1"; body="${3:-}" + if [ "$method" = POST ]; then + printf '%s\n' "$body" | jq -c . >> "$calls" + GCR_LAST_HTTP=500 + return 1 + fi + printf 'lookup-failed\n' >> "$GCR_STATE_DIR/lookups" + GCR_LAST_HTTP=503 + return 1 +} +gcr_alloc 9012 1 hectic-lab/util.nix '["nix"]' +test "$RESPONSE_BODY" = 'VM creation pending recovery' +test "$(wc -l < "$calls" | tr -d ' ')" = 1 +ambiguous_rec="$(gcr_record_get 9012 1)" +test "$(gcr_record_field "$ambiguous_rec" status)" = create_ambiguous +test "$(gcr_record_field "$ambiguous_rec" budget_rate)" = 0.004 +test "$(gcr_count_active)" = 1 +test "$(cat "$GCR_STATE_DIR/budget/$(date -u '+%Y-%m')")" = '0.0320' +gcr_sweep_create_ambiguous +test "$(gcr_record_field "$(gcr_record_get 9012 1)" status)" = create_ambiguous +test "$(wc -l < "$calls" | tr -d ' ')" = 1 + +gcr_hcloud_req() { + test "$1" = GET + GCR_LAST_BODY="$GCR_STATE_DIR/last-body.json" + GCR_LAST_HTTP=200 + printf '%s\n' '{"servers":[{"id":9012,"name":"gcr-9012-1","server_type":{"name":"cx23"},"labels":{"gitea-runner-controller":"managed","gcr.job-id":"9012","gcr.run-attempt":"1","gcr.label":"nix","gcr.location":"nbg1","gcr.arch":"amd64"}}]}' > "$GCR_LAST_BODY" +} +gcr_vm_destroy() { printf 'unexpected-destroy %s\n' "$1" >> "$GCR_STATE_DIR/ambiguity-destroys"; return 1; } +gcr_sweep_create_ambiguous +recovered_rec="$(gcr_record_get 9012 1)" +test "$(gcr_record_field "$recovered_rec" status)" = pending_vm +test "$(gcr_record_field "$recovered_rec" vm_id)" = 9012 +test "$(gcr_record_field "$recovered_rec" server_type)" = cx23 +test "$(gcr_record_field "$recovered_rec" budget_rate)" = 0.004 +test "$(cat "$GCR_STATE_DIR/budget/$(date -u '+%Y-%m')")" = '0.0320' +test ! -e "$GCR_STATE_DIR/ambiguity-destroys" +gcr_record_del 9012 1 +rm -f "$GCR_STATE_DIR/budget/$(date -u '+%Y-%m')" + +# Fallback candidates exceeding remaining budget never reach Hetzner. +rm -f "$calls" "$GCR_STATE_DIR/budget/$(date -u '+%Y-%m')" +export GCR_BUDGET_EUR_MONTHLY='0.05' +gcr_hcloud_req() { + method="$1"; path="$2"; body="${3:-}" + GCR_LAST_BODY="$GCR_STATE_DIR/last-body.json" + if [ "$method" = GET ]; then + printf '{"servers":[]}\n' > "$GCR_LAST_BODY" + GCR_LAST_HTTP=200 + return 0 + fi + test "$path" = /servers + printf '%s\n' "$body" | jq -c . >> "$calls" + GCR_LAST_HTTP=412 + return 1 +} +gcr_alloc 9003 1 hectic-lab/util.nix '["nix"]' +test "$RESPONSE_BODY" = 'VM creation failed' +test "$(wc -l < "$calls" | tr -d ' ')" = '3' +jq -e -s 'all(.server_type == "cx23")' "$calls" >/dev/null +test "$(cat "$GCR_STATE_DIR/budget/$(date -u '+%Y-%m')")" = '0.0000' + +# Deferred allocation keeps cleanup ownership and reservation across failed +# DELETE, failed post-DELETE state update, and cleanup-owned 404 retry. +rm -f "$calls" "$GCR_STATE_DIR/budget/$(date -u '+%Y-%m')" +export GCR_BUDGET_EUR_MONTHLY='10' +deferred='{"job_id":"9005","run_attempt":"1","repo":"hectic-lab/util.nix","label":"nix","created_at":"0","ttl_min":null,"vm_id":"","vm_name":"","status":"deferred"}' +printf '%s\n' "$deferred" > "$GCR_STATE_DIR/jobs/9005-1.json" +gcr_gitea_job_state() { printf queued; } +record_put_count="$GCR_STATE_DIR/record-put-count" +gcr_record_put() { + count="$(cat "$record_put_count" 2>/dev/null || echo 0)" + count=$((count + 1)) + printf '%s\n' "$count" > "$record_put_count" + case "$count" in 1|3|5|8) return 1 ;; esac + tmp="$(mktemp "$(dirname "$(gcr_record_path "$1" "$2")")/.tmp.XXXXXX")" + printf '%s\n' "$3" > "$tmp" + mv -f "$tmp" "$(gcr_record_path "$1" "$2")" +} +gcr_hcloud_req() { + test "$1" = POST + printf '%s\n' "$3" | jq -c . >> "$calls" + GCR_LAST_BODY="$GCR_STATE_DIR/last-body.json" + printf '{"server":{"id":9005}}\n' > "$GCR_LAST_BODY" +} +gcr_vm_destroy() { + printf 'destroy-failed %s\n' "$1" >> "$GCR_STATE_DIR/destroy-calls" + return 1 +} +gcr_alloc_deferred 9005 1 +cleanup_rec="$(gcr_record_get 9005 1)" +test "$(gcr_record_field "$cleanup_rec" status)" = cleanup_pending +test "$(gcr_record_field "$cleanup_rec" vm_id)" = 9005 +test "$(gcr_record_field "$cleanup_rec" budget_rate)" = 0.004 +test "$(cat "$GCR_STATE_DIR/budget/$(date -u '+%Y-%m')")" = '0.0320' + +gcr_vm_destroy() { printf 'deleted %s\n' "$1" >> "$GCR_STATE_DIR/destroy-calls"; } +gcr_sweep_cleanup_pending +cleanup_rec="$(gcr_record_get 9005 1)" +test "$(gcr_record_field "$cleanup_rec" status)" = cleanup_pending +test "$(gcr_record_field "$cleanup_rec" cleanup_vm_destroyed)" = false +test "$(cat "$GCR_STATE_DIR/budget/$(date -u '+%Y-%m')")" = '0.0320' +gcr_vm_destroy() { + printf 'absent %s\n' "$1" >> "$GCR_STATE_DIR/destroy-calls" + GCR_LAST_HTTP=404 + return 1 +} +gcr_sweep_cleanup_pending +cleanup_rec="$(gcr_record_get 9005 1)" +test "$(gcr_record_field "$cleanup_rec" cleanup_vm_destroyed)" = true +test "$(cat "$GCR_STATE_DIR/budget/$(date -u '+%Y-%m')")" = '0.0000' +gcr_sweep_cleanup_pending +test ! -e "$(gcr_record_path 9005 1)" +test "$(cat "$GCR_STATE_DIR/budget/$(date -u '+%Y-%m')")" = '0.0000' +grep -q '^destroy-failed 9005$' "$GCR_STATE_DIR/destroy-calls" +grep -q '^deleted 9005$' "$GCR_STATE_DIR/destroy-calls" +grep -q '^absent 9005$' "$GCR_STATE_DIR/destroy-calls" +test "$(grep -c '^absent 9005$' "$GCR_STATE_DIR/destroy-calls")" = 1 + +# Confirmed ambiguous absence uses same monotonic refund claim. Failed state +# persistence after credit cannot make retry subtract reservation twice. +gcr_budget_add 0.004 480 +ambiguous_absent='{"job_id":"9013","run_attempt":"1","repo":"hectic-lab/util.nix","label":"nix","created_at":"0","ttl_min":480,"vm_id":"","vm_name":"gcr-9013-1","server_type":"cx23","budget_rate":"0.004","candidate_location":"nbg1","candidate_arch":"amd64","bootstrapped":false,"status":"create_ambiguous"}' +printf '%s\n' "$ambiguous_absent" > "$GCR_STATE_DIR/jobs/9013-1.json" +gcr_hcloud_req() { + test "$1" = GET + GCR_LAST_BODY="$GCR_STATE_DIR/last-body.json" + GCR_LAST_HTTP=200 + printf '{"servers":[]}\n' > "$GCR_LAST_BODY" +} +gcr_sweep_create_ambiguous +absence_rec="$(gcr_record_get 9013 1)" +test "$(gcr_record_field "$absence_rec" status)" = cleanup_pending +test "$(gcr_record_field "$absence_rec" cleanup_vm_destroyed)" = true +test "$(cat "$GCR_STATE_DIR/budget/$(date -u '+%Y-%m')")" = '0.0000' +gcr_sweep_cleanup_pending +test ! -e "$(gcr_record_path 9013 1)" +test "$(cat "$GCR_STATE_DIR/budget/$(date -u '+%Y-%m')")" = '0.0000' + +# Reservation write failure aborts before any Hetzner request. +rm -f "$calls" +rm -rf "$GCR_STATE_DIR/budget" +printf 'not-a-directory\n' > "$GCR_STATE_DIR/budget" +gcr_alloc 9006 1 hectic-lab/util.nix '["nix"]' +test "$RESPONSE_BODY" = 'VM creation failed' +test ! -e "$calls" +test ! -e "$(gcr_record_path 9006 1)" diff --git a/test/package/gitea-runner-controller/test/idle-reuse.sh b/test/package/gitea-runner-controller/test/idle-reuse.sh index efd53f07..a2d8adcb 100644 --- a/test/package/gitea-runner-controller/test/idle-reuse.sh +++ b/test/package/gitea-runner-controller/test/idle-reuse.sh @@ -19,7 +19,11 @@ gcr_now_epoch() { printf '%s' "$NOW"; } calls="$GCR_STATE_DIR/calls" gcr_budget_add() { printf 'budget\n' >> "$calls"; } gcr_gitea_registration_token() { printf 'token'; printf 'token\n' >> "$calls"; } -gcr_vm_create() { printf 'create\n' >> "$calls"; printf '99'; } +gcr_vm_create() { + printf 'create\n' >> "$calls" + gcr_budget_add 0.032 180 + printf '99 cx53 0.032' +} gcr_vm_destroy() { printf 'destroy vm=%s\n' "$1" >> "$calls"; } gcr_vm_runner_service() { printf 'runner %s vm=%s\n' "$2" "$1" >> "$calls"; } gcr_gitea_runner_disabled() { printf 'runner-disabled %s %s\n' "$2" "$3" >> "$calls"; } @@ -270,6 +274,7 @@ set -eu . "$LOG_SH" . "$STATE_SH" . "$DECIDE_SH" +. "$HCLOUD_SH" . "$WEBHOOK_SH" gcr_now_epoch() { printf '2000'; } gcr_budget_add() { printf 'budget %s\n' "$1" >> "$GCR_STATE_DIR/admission-calls"; } @@ -277,7 +282,8 @@ gcr_gitea_registration_token() { printf token; } gcr_vm_create() { sleep 1 printf 'create %s\n' "$6" >> "$GCR_STATE_DIR/admission-calls" - printf '%s' "$6" + gcr_budget_add 0.032 180 + printf '%s cx53 0.032' "$6" } gcr_vm_destroy() { :; } gcr_state_init diff --git a/test/package/gitea-runner-controller/test/reap-diagnostics.sh b/test/package/gitea-runner-controller/test/reap-diagnostics.sh index 3f3a753a..2a01d81d 100644 --- a/test/package/gitea-runner-controller/test/reap-diagnostics.sh +++ b/test/package/gitea-runner-controller/test/reap-diagnostics.sh @@ -63,6 +63,29 @@ gcr_reap_finished_jobs test "$(gcr_record_get 101 1)" = "$idle_once" test ! -e "$(gcr_record_path 102 1)" +# Reaper keeps cleanup ownership after DELETE failure and retries next sweep. +record_delete_fail='{"job_id":"104","run_attempt":"1","repo":"hinterland/hearth","label":"nix","created_at":"1","ttl_min":480,"vm_id":44,"vm_name":"gcr-104-1","bootstrapped":true,"status":"vm_active"}' +gcr_record_put 104 1 "$record_delete_fail" +gcr_gitea_job_state() { + case "$2" in + 104) printf 'completed:failure' ;; + *) return 1 ;; + esac +} +gcr_vm_destroy() { + printf 'destroy-failed vm=%s\n' "$1" >> "$calls" + return 1 +} +gcr_reap_finished_jobs +test "$(gcr_record_field "$(gcr_record_get 104 1)" status)" = cleanup_pending +test "$(gcr_count_active)" = 1 + +gcr_vm_destroy() { printf 'destroy-retry vm=%s\n' "$1" >> "$calls"; } +gcr_sweep_cleanup_pending +test ! -e "$(gcr_record_path 104 1)" +grep -q '^destroy-failed vm=44$' "$calls" +grep -q '^destroy-retry vm=44$' "$calls" + calls_ip_fail="$GCR_STATE_DIR/calls-ip-fail" calls="$calls_ip_fail" record_ip_fail='{"job_id":"103","run_attempt":"1","repo":"hinterland/hearth","label":"gross-nix-x86-perf","created_at":"1","ttl_min":480,"vm_id":43,"vm_name":"gcr-103-1","bootstrapped":true,"status":"vm_active"}' @@ -79,6 +102,10 @@ gcr_vm_public_ip() { return 1 } +gcr_vm_destroy() { + printf 'destroy vm=%s\n' "$1" >> "$calls" +} + gcr_reap_finished_jobs grep -q 'diag vm=43 ip= job=103 reason=completed:failure' "$calls_ip_fail" diff --git a/test/package/gitea-runner-controller/test/webhook-diagnostics.sh b/test/package/gitea-runner-controller/test/webhook-diagnostics.sh index c8994115..dd96f49e 100644 --- a/test/package/gitea-runner-controller/test/webhook-diagnostics.sh +++ b/test/package/gitea-runner-controller/test/webhook-diagnostics.sh @@ -71,6 +71,29 @@ grep -q 'diag vm=53 ip= job=203 reason=completed:failure' "$calls_ip_fail" grep -q 'destroy vm=53' "$calls_ip_fail" test ! -e "$(gcr_record_path 203 1)" +# Terminal webhook retains ownership and capacity until DELETE succeeds. +record_delete_fail='{"job_id":"204","run_attempt":"1","repo":"hinterland/hearth","label":"nix","created_at":"1","ttl_min":480,"vm_id":54,"vm_name":"gcr-204-1","bootstrapped":true,"status":"vm_active"}' +gcr_record_put 204 1 "$record_delete_fail" +gcr_vm_destroy() { + printf 'destroy-failed vm=%s\n' "$1" >> "$calls_ip_fail" + return 1 +} +gcr_deallocate 204 1 completed:failure +cleanup_rec="$(gcr_record_get 204 1)" +test "$(gcr_record_field "$cleanup_rec" status)" = cleanup_pending +test "$(gcr_count_active)" = 1 +test "$(gcr_count_active_repo hinterland/hearth)" = 1 + +gcr_vm_destroy() { printf 'destroy-retry vm=%s\n' "$1" >> "$calls_ip_fail"; } +key="$(gcr_alloc_key 204 1)" +gcr_lock_acquire "$key" +gcr_vm_cleanup_pending 204 1 "$(gcr_record_get 204 1)" +gcr_lock_release "$key" +test ! -e "$(gcr_record_path 204 1)" +test "$(gcr_count_active)" = 0 +grep -q '^destroy-failed vm=54$' "$calls_ip_fail" +grep -q '^destroy-retry vm=54$' "$calls_ip_fail" + # Gitea emits zero-based run_attempt values for initial workflow jobs. gcr_read_request() { gcr_hdr_event_type=workflow_job