hinterland/hearth
2
2
Fork 1
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix: runners
runner nix smoke / nix label and flake smoke (push) Failing after 1m1s
Details

Browse Source
This commit is contained in:
yukkop
2026-09-11 05:58:04 +00:00
parent f0317efb6e
commit 4a88e7c6e8
9 changed files with 736 additions and 99 deletions
Download Patch File Download Diff File Expand all files Collapse all files
package/gitea-runner-controller/controller.sh
+150 -37
View File
@@ -40,10 +40,16 @@ gcr_sweep_ttl() {
fi
gcr_log info --ns=sweep "idle slot expired job=$job_id vm=$vm_id"
if [ -n "$vm_id" ] && [ "$vm_id" != "null" ] && [ "$vm_id" != "0" ]; then
gcr_vm_destroy "$vm_id" || true
gcr_event "vm-destroyed" "$job_id" "{\"vm_id\":$vm_id,\"reason\":\"idle-expired\"}"
if gcr_vm_cleanup_start "$job_id" "$attempt" "$rec" idle-expired false; then
gcr_event "vm-destroyed" "$job_id" \
"{\"vm_id\":$vm_id,\"reason\":\"idle-expired\"}"
else
gcr_event "vm-cleanup-pending" "$job_id" \
"{\"vm_id\":$vm_id,\"reason\":\"idle-expired\"}"
fi
else
gcr_record_del "$job_id" "$attempt"
fi
gcr_record_del "$job_id" "$attempt"
gcr_lock_release idle-pool
continue
fi
@@ -78,11 +84,17 @@ gcr_sweep_ttl() {
if [ -n "$vm_id" ] && [ "$vm_id" != "null" ] && [ "$vm_id" != "0" ]; then
ip="$(gcr_vm_public_ip "$vm_id" || true)"
gcr_vm_collect_diagnostics "$vm_id" "$ip" "$job_id" ttl || true
gcr_vm_destroy "$vm_id" || true
gcr_event "vm-destroyed" "$job_id" "{\"vm_id\":$vm_id,\"reason\":\"ttl\"}"
if gcr_vm_cleanup_start "$job_id" "$attempt" "$rec" ttl false; then
gcr_event "vm-destroyed" "$job_id" \
"{\"vm_id\":$vm_id,\"reason\":\"ttl\"}"
else
gcr_event "vm-cleanup-pending" "$job_id" \
"{\"vm_id\":$vm_id,\"reason\":\"ttl\"}"
fi
else
gcr_record_del "$job_id" "$attempt"
fi
gcr_event "job-ttl-expired" "$job_id" "{\"age\":$age}"
gcr_record_del "$job_id" "$attempt"
gcr_lock_release "$key"
fi
done
@@ -108,8 +120,20 @@ gcr_sweep_orphan_vms() {
if ! gcr_record_exists_for_vm_id "$vm_id"; then
gcr_log warn --ns=sweep "orphan VM $vm_id job=$jid attempt=$att -> destroy"
gcr_vm_destroy "$vm_id" || true
gcr_event "orphan-vm-destroyed" "${jid:-unknown}" "{\"vm_id\":$vm_id}"
cleanup_job="${jid:-orphan-$vm_id}"
cleanup_attempt="${att:-0}"
cleanup_rec="$(jq -n --arg j "$cleanup_job" --arg a "$cleanup_attempt" \
--arg v "$vm_id" --arg vn "$(printf '%s' "$vm" | jq -r '.name // ""')" \
'{job_id:$j, run_attempt:$a, repo:"", label:"", created_at:"0",
ttl_min:0, vm_id:($v|tonumber), vm_name:$vn,
bootstrapped:false, status:"cleanup_pending"}')"
if gcr_vm_cleanup_start "$cleanup_job" "$cleanup_attempt" \
"$cleanup_rec" orphan false; then
gcr_event "orphan-vm-destroyed" "$cleanup_job" "{\"vm_id\":$vm_id}"
else
gcr_event "vm-cleanup-pending" "$cleanup_job" \
"{\"vm_id\":$vm_id,\"reason\":\"orphan\"}"
fi
fi
i=$((i + 1))
done
@@ -149,7 +173,7 @@ gcr_alloc_deferred() {
return 0
}
set -- $profile
server_type="$1"; ttl_min="$2"; rate="$3"
server_type="$1"; ttl_min="$2"
gcr_lock_acquire admission || {
gcr_lock_release "$key"
@@ -190,11 +214,6 @@ gcr_alloc_deferred() {
return 0
fi
gcr_budget_can_add "$rate" "$ttl_min" || {
gcr_lock_release admission
gcr_lock_release "$key"
return 0
}
reg_token="$(gcr_gitea_registration_token "$repo")" || {
gcr_lock_release admission
gcr_lock_release "$key"
@@ -203,23 +222,19 @@ gcr_alloc_deferred() {
vm_name="gcr-${job_id}-${attempt}"
created_at="$(gcr_now_epoch)"
vm_id="$(gcr_vm_create "$vm_name" "$label" "$server_type" "$ttl_min" \
"$reg_token" "$job_id" "$attempt" "$repo")" && [ -n "$vm_id" ] || {
create_status=0
created="$(gcr_vm_create "$vm_name" "$label" "$server_type" "$ttl_min" \
"$reg_token" "$job_id" "$attempt" "$repo")" || create_status="$?"
if [ "$create_status" -ne 0 ] || [ -z "$created" ]; then
gcr_lock_release admission
gcr_lock_release "$key"
return 0
}
gcr_budget_add "$rate" "$ttl_min"
rec="$(jq -n --arg j "$job_id" --arg a "$attempt" --arg r "$repo" \
--arg l "$label" --arg t "$created_at" --arg v "$vm_id" \
--arg vn "$vm_name" --arg ttl "$ttl_min" \
'{job_id:$j, run_attempt:$a, repo:$r, label:$l,
created_at:$t, ttl_min:($ttl|tonumber), vm_id:($v|tonumber),
vm_name:$vn, bootstrapped:false, status:"pending_vm"}')"
if ! gcr_record_put "$job_id" "$attempt" "$rec"; then
gcr_vm_destroy "$vm_id" || true
fi
set -- $created
vm_id="$1"; actual_server_type="$2"; actual_rate="$3"
if ! gcr_vm_record_created "$job_id" "$attempt" "$repo" "$label" \
"$created_at" "$vm_id" "$vm_name" "$ttl_min" \
"$actual_server_type" "$actual_rate"; then
gcr_lock_release admission
gcr_lock_release "$key"
return 0
@@ -230,6 +245,83 @@ gcr_alloc_deferred() {
gcr_log info --ns=alloc "deferred job=$job_id allocated vm=$vm_id"
}
gcr_sweep_cleanup_pending() {
for f in $(gcr_active_records); do
rec="$(cat "$f")"
[ "$(gcr_record_field "$rec" status)" = "cleanup_pending" ] || continue
job_id="$(gcr_record_field "$rec" job_id)"
attempt="$(gcr_record_field "$rec" run_attempt)"
key="$(gcr_alloc_key "$job_id" "$attempt")"
gcr_lock_acquire "$key" || continue
if ! gcr_lock_acquire admission; then
gcr_lock_release "$key"
continue
fi
rec="$(gcr_record_get "$job_id" "$attempt")"
if [ "$(gcr_record_field "$rec" status)" = "cleanup_pending" ]; then
gcr_vm_cleanup_pending "$job_id" "$attempt" "$rec" || true
fi
gcr_lock_release admission
gcr_lock_release "$key"
done
}
gcr_sweep_create_ambiguous() {
for f in $(gcr_active_records); do
rec="$(cat "$f")"
[ "$(gcr_record_field "$rec" status)" = "create_ambiguous" ] || continue
job_id="$(gcr_record_field "$rec" job_id)"
attempt="$(gcr_record_field "$rec" run_attempt)"
key="$(gcr_alloc_key "$job_id" "$attempt")"
gcr_lock_acquire "$key" || continue
if ! gcr_lock_acquire admission; then
gcr_lock_release "$key"
continue
fi
rec="$(gcr_record_get "$job_id" "$attempt")"
if [ "$(gcr_record_field "$rec" status)" != "create_ambiguous" ]; then
gcr_lock_release admission
gcr_lock_release "$key"
continue
fi
find_status=0
found_vm_id="$(gcr_vm_find_created \
"$(gcr_record_field "$rec" vm_name)" "$job_id" "$attempt" \
"$(gcr_record_field "$rec" label)" \
"$(gcr_record_field "$rec" server_type)" \
"$(gcr_record_field "$rec" candidate_location)" \
"$(gcr_record_field "$rec" candidate_arch)")" || find_status="$?"
case "$find_status" in
0)
rec="$(printf '%s' "$rec" | jq -c --arg vm "$found_vm_id" \
'.vm_id = ($vm | tonumber)
| .status = "pending_vm"
| .bootstrapped = false
| del(.create_http, .create_curl_status,
.candidate_location, .candidate_arch)')"
if gcr_record_put "$job_id" "$attempt" "$rec"; then
gcr_event "vm-create-recovered" "$job_id" \
"{\"vm_id\":$found_vm_id,\"label\":\"$(gcr_record_field "$rec" label)\"}"
fi
;;
1)
rec="$(printf '%s' "$rec" | jq -c \
'.status = "cleanup_pending"
| .cleanup_reason = "ambiguous-create-absent"
| .cleanup_refund_budget = true
| .cleanup_vm_destroyed = true
| .cleanup_budget_released = false')"
if gcr_record_put "$job_id" "$attempt" "$rec"; then
gcr_vm_cleanup_pending "$job_id" "$attempt" "$rec" || true
fi
;;
2) ;;
esac
gcr_lock_release admission
gcr_lock_release "$key"
done
}
gcr_retry_deferred() {
for f in $(gcr_active_records); do
rec="$(cat "$f")"
@@ -343,10 +435,17 @@ gcr_bootstrap_pending() {
gcr_vm_collect_diagnostics "$vm_id" "$ip" "$job_id" "$state" || true
;;
esac
gcr_vm_destroy "$vm_id" || true
gcr_event "vm-destroyed" "$job_id" "{\"vm_id\":$vm_id,\"reason\":\"pending-job-completed\",\"state\":\"$state\"}"
if gcr_vm_cleanup_start "$job_id" "$attempt" "$rec" \
pending-job-completed false; then
gcr_event "vm-destroyed" "$job_id" \
"{\"vm_id\":$vm_id,\"reason\":\"pending-job-completed\",\"state\":\"$state\"}"
else
gcr_event "vm-cleanup-pending" "$job_id" \
"{\"vm_id\":$vm_id,\"reason\":\"pending-job-completed\"}"
fi
else
gcr_record_del "$job_id" "$attempt"
fi
gcr_record_del "$job_id" "$attempt"
gcr_lock_release "$key"
continue
;;
@@ -418,10 +517,15 @@ gcr_reap_finished_jobs() {
if ! gcr_gitea_runner_disabled "$repo" "$runner_name" true \
|| ! gcr_vm_runner_service "$vm_id" stop; then
gcr_lock_release idle-pool
gcr_vm_destroy "$vm_id" || true
gcr_record_del "$job_id" "$attempt"
if gcr_vm_cleanup_start "$job_id" "$attempt" "$rec" \
idle-stop-failed false; then
gcr_event "vm-destroyed" "$job_id" \
"{\"vm_id\":$vm_id,\"reason\":\"idle-stop-failed\",\"via\":\"reconcile\"}"
else
gcr_event "vm-cleanup-pending" "$job_id" \
"{\"vm_id\":$vm_id,\"reason\":\"idle-stop-failed\",\"via\":\"reconcile\"}"
fi
gcr_lock_release "$key"
gcr_event "vm-destroyed" "$job_id" "{\"vm_id\":$vm_id,\"reason\":\"idle-stop-failed\",\"via\":\"reconcile\"}"
continue
fi
gcr_record_put "$job_id" "$attempt" "$idle_rec"
@@ -442,10 +546,17 @@ gcr_reap_finished_jobs() {
gcr_vm_collect_diagnostics "$vm_id" "$ip" "$job_id" "$state" || true
;;
esac
gcr_vm_destroy "$vm_id" || true
gcr_event "vm-destroyed" "$job_id" "{\"vm_id\":$vm_id,\"reason\":\"job-completed\",\"state\":\"$state\"}"
if gcr_vm_cleanup_start "$job_id" "$attempt" "$rec" \
job-completed false; then
gcr_event "vm-destroyed" "$job_id" \
"{\"vm_id\":$vm_id,\"reason\":\"job-completed\",\"state\":\"$state\"}"
else
gcr_event "vm-cleanup-pending" "$job_id" \
"{\"vm_id\":$vm_id,\"reason\":\"job-completed\",\"state\":\"$state\"}"
fi
else
gcr_record_del "$job_id" "$attempt"
fi
gcr_record_del "$job_id" "$attempt"
gcr_lock_release "$key"
;;
esac
@@ -453,6 +564,8 @@ gcr_reap_finished_jobs() {
}
gcr_tick() {
gcr_sweep_create_ambiguous
gcr_sweep_cleanup_pending
gcr_sweep_ttl
gcr_reap_finished_jobs
gcr_sweep_orphan_vms
package/gitea-runner-controller/decide.sh
+18 -4
View File
@@ -41,7 +41,14 @@ gcr_label_ttl() {
gcr_label_candidates() {
label="$1"
case "$label" in
ubuntu-latest|gross-x86)
ubuntu-latest)
printf '%s\n' \
'cx23 nbg1 amd64' 'cx23 fsn1 amd64' 'cx23 hel1 amd64' \
'cx33 nbg1 amd64' 'cx33 fsn1 amd64' 'cx33 hel1 amd64' \
'cx43 nbg1 amd64' 'cx43 fsn1 amd64' 'cx43 hel1 amd64' \
'cx53 nbg1 amd64' 'cx53 fsn1 amd64' 'cx53 hel1 amd64'
;;
gross-x86)
printf '%s\n' \
'cx53 nbg1 amd64' 'cx53 fsn1 amd64' 'cx53 hel1 amd64' \
'cx43 nbg1 amd64' 'cx43 fsn1 amd64' 'cx43 hel1 amd64' \
@@ -65,7 +72,14 @@ gcr_label_candidates() {
'cax41 nbg1 arm64' 'cax41 fsn1 arm64' 'cax41 hel1 arm64' \
'cx43 nbg1 amd64' 'cx43 fsn1 amd64' 'cx43 hel1 amd64'
;;
nix|gross-nix-x86)
nix)
printf '%s\n' \
'cx23 nbg1 amd64' 'cx23 fsn1 amd64' 'cx23 hel1 amd64' \
'cx33 nbg1 amd64' 'cx33 fsn1 amd64' 'cx33 hel1 amd64' \
'cx43 nbg1 amd64' 'cx43 fsn1 amd64' 'cx43 hel1 amd64' \
'cx53 nbg1 amd64' 'cx53 fsn1 amd64' 'cx53 hel1 amd64'
;;
gross-nix-x86)
printf '%s\n' \
'cx53 nbg1 amd64' 'cx53 fsn1 amd64' 'cx53 hel1 amd64' \
'cx43 nbg1 amd64' 'cx43 fsn1 amd64' 'cx43 hel1 amd64' \
@@ -148,7 +162,7 @@ gcr_count_active() {
for f in $(gcr_active_records); do
status="$(gcr_record_field "$(cat "$f")" status)"
case "$status" in
pending_vm|vm_active) count=$((count + 1)) ;;
pending_vm|vm_active|cleanup_pending|create_ambiguous) count=$((count + 1)) ;;
esac
done
printf '%s' "$count"
@@ -160,7 +174,7 @@ gcr_count_active_repo() {
for f in $(gcr_active_records); do
rec="$(cat "$f")"
case "$(gcr_record_field "$rec" status)" in
pending_vm|vm_active) ;;
pending_vm|vm_active|cleanup_pending|create_ambiguous) ;;
*) continue ;;
esac
[ "$(gcr_record_field "$rec" repo)" = "$repo" ] && count=$((count + 1))
package/gitea-runner-controller/hcloud.sh
+179 -6
View File
@@ -36,6 +36,7 @@ gcr_hcloud_req() {
method="$1"; path="$2"; body="${3:-}"
token="$(gcr_hcloud_token)" || return 1
GCR_LAST_BODY="$(mktemp "${TMPDIR:-/tmp}/gcr-resp.XXXXXX")"
curl_status=0
if [ -n "$body" ]; then
code="$(printf '%s' "$body" | curl -sS -X "$method" \
-H "Authorization: Bearer $token" \
@@ -43,13 +44,20 @@ gcr_hcloud_req() {
--data-binary @- \
-o "$GCR_LAST_BODY" \
-w '%{http_code}' \
"$GCR_API$path")"
"$GCR_API$path")" || curl_status="$?"
else
code="$(curl -sS -X "$method" \
-H "Authorization: Bearer $token" \
-o "$GCR_LAST_BODY" \
-w '%{http_code}' \
"$GCR_API$path")"
"$GCR_API$path")" || curl_status="$?"
fi
case "$code" in ''|*[!0-9]*) code=000 ;; esac
GCR_LAST_CURL_STATUS="$curl_status"
GCR_LAST_HTTP="$code"
if [ "$curl_status" -ne 0 ]; then
gcr_log warn --ns=hcloud "transport failed path=$path curl=$curl_status http=$code"
return 1
fi
case "$code" in 2??) return 0 ;; esac
gcr_log warn --ns=hcloud "request failed path=$path http=$code body=$(head -c 200 "$GCR_LAST_BODY" | gcr_redact)"
@@ -62,6 +70,64 @@ gcr_vm_list_managed() {
fi
}
# Exact deterministic create identity. Exit 0 = one match (prints id),
# 1 = confirmed absent, 2 = lookup failed or identity invariant violated.
gcr_vm_find_created() {
find_name="$1"; find_job="$2"; find_attempt="$3"; find_label="$4"
find_type="$5"; find_location="$6"; find_arch="$7"
if ! gcr_hcloud_req GET "/servers?name=$find_name"; then
return 2
fi
find_matches="$(jq -c \
--arg name "$find_name" --arg job "$find_job" --arg attempt "$find_attempt" \
--arg label "$find_label" --arg type "$find_type" \
--arg location "$find_location" --arg arch "$find_arch" \
'[.servers[] | select(
.name == $name
and .labels["gitea-runner-controller"] == "managed"
and .labels["gcr.job-id"] == $job
and .labels["gcr.run-attempt"] == $attempt
and .labels["gcr.label"] == $label
and .labels["gcr.location"] == $location
and .labels["gcr.arch"] == $arch
and ((.server_type.name // .server_type) == $type))]' \
"$GCR_LAST_BODY")" || return 2
find_count="$(printf '%s' "$find_matches" | jq 'length')" || return 2
case "$find_count" in
0) return 1 ;;
1) printf '%s' "$find_matches" | jq -r '.[0].id' ;;
*) return 2 ;;
esac
}
gcr_create_explicitly_rejected() {
case "$1" in
400|401|403|404|405|409|412|422|423) return 0 ;;
*) return 1 ;;
esac
}
gcr_vm_record_create_ambiguous() {
ambiguous_name="$1"; ambiguous_label="$2"; ambiguous_ttl="$3"
ambiguous_job="$4"; ambiguous_attempt="$5"; ambiguous_repo="$6"
ambiguous_type="$7"; ambiguous_rate="$8"; ambiguous_location="$9"
shift 9; ambiguous_arch="$1"; ambiguous_http="$2"; ambiguous_curl="$3"
ambiguous_rec="$(jq -n --arg j "$ambiguous_job" --arg a "$ambiguous_attempt" \
--arg r "$ambiguous_repo" --arg l "$ambiguous_label" \
--arg t "$(gcr_now_epoch)" --arg vn "$ambiguous_name" \
--arg ttl "$ambiguous_ttl" --arg st "$ambiguous_type" \
--arg rate "$ambiguous_rate" --arg loc "$ambiguous_location" \
--arg arch "$ambiguous_arch" --arg http "$ambiguous_http" \
--arg curl "$ambiguous_curl" \
'{job_id:$j, run_attempt:$a, repo:$r, label:$l,
created_at:$t, ttl_min:($ttl|tonumber), vm_id:"", vm_name:$vn,
server_type:$st, budget_rate:$rate, candidate_location:$loc,
candidate_arch:$arch, create_http:$http, create_curl_status:$curl,
bootstrapped:false,
status:"create_ambiguous"}')"
gcr_record_put "$ambiguous_job" "$ambiguous_attempt" "$ambiguous_rec"
}
gcr_vm_build_userdata() {
vm_name="$1"; label="$2"; reg_token="$3"
@@ -163,9 +229,10 @@ runcmd:
}
# gcr_vm_create NAME LABEL SERVER_TYPE TTL_MIN REG_TOKEN JOB_ID ATTEMPT REPO
# Prints new server id.
# Caller holds admission lock. Reserves each affordable candidate before its
# create request; prints new server id, actual server type, and reserved rate.
gcr_vm_create() {
vm_name="$1"; label="$2"; server_type="$3"; ttl_min="$4"
vm_name="$1"; label="$2"; ttl_min="$4"
reg_token="$5"; job_id="$6"; attempt="$7"; repo="$8"
ttl_min="$(gcr_label_ttl "$label")" || return 1
@@ -174,8 +241,18 @@ gcr_vm_create() {
while read -r candidate_type candidate_loc candidate_arch; do
[ -n "${candidate_type:-}" ] || continue
candidate_n=$((candidate_n + 1))
candidate_rate="$(gcr_server_hourly_rate "$candidate_type")" || continue
if ! gcr_budget_can_add "$candidate_rate" "$ttl_min"; then
gcr_log info --ns=hcloud "skip candidate[$candidate_n] label=$label type=$candidate_type over budget"
continue
fi
if ! gcr_budget_add "$candidate_rate" "$ttl_min"; then
gcr_log error --ns=hcloud "budget reservation write failed label=$label type=$candidate_type"
return 1
fi
image_id="$(gcr_image_id_for_arch "$candidate_arch" "$label")" || {
gcr_log warn --ns=hcloud "skip candidate[$candidate_n] label=$label arch=$candidate_arch no image"
gcr_budget_sub "$candidate_rate" "$ttl_min" || return 1
continue
}
payload="$(jq -n \
@@ -203,9 +280,34 @@ gcr_vm_create() {
"gcr.created-at":$ts, "gcr.ttl-min":$ttl}}')"
gcr_log info --ns=hcloud "try candidate[$candidate_n] label=$label type=$candidate_type arch=$candidate_arch loc=$candidate_loc"
if gcr_hcloud_req POST /servers "$payload"; then
jq -r '.server.id' "$GCR_LAST_BODY"
printf '%s %s %s\n' \
"$(jq -r '.server.id' "$GCR_LAST_BODY")" "$candidate_type" "$candidate_rate"
return 0
fi
create_http="${GCR_LAST_HTTP:-000}"
create_curl="${GCR_LAST_CURL_STATUS:-0}"
find_status=0
found_vm_id="$(gcr_vm_find_created "$vm_name" "$job_id" "$attempt" \
"$label" "$candidate_type" "$candidate_loc" "$candidate_arch")" \
|| find_status="$?"
if [ "$find_status" -eq 0 ]; then
printf '%s %s %s\n' "$found_vm_id" "$candidate_type" "$candidate_rate"
return 0
fi
if [ "$find_status" -eq 1 ] \
&& gcr_create_explicitly_rejected "$create_http"; then
if ! gcr_budget_sub "$candidate_rate" "$ttl_min"; then
gcr_log error --ns=hcloud "budget reservation rollback failed label=$label type=$candidate_type"
return 1
fi
else
if ! gcr_vm_record_create_ambiguous "$vm_name" "$label" "$ttl_min" \
"$job_id" "$attempt" "$repo" "$candidate_type" "$candidate_rate" \
"$candidate_loc" "$candidate_arch" "$create_http" "$create_curl"; then
gcr_log error --ns=hcloud "cannot persist ambiguous create job=$job_id type=$candidate_type"
fi
return 2
fi
if [ "$candidate_n" -le 3 ]; then
sleep 5
else
@@ -217,7 +319,7 @@ EOF
return 1
}
# gcr_vm_destroy SERVER_ID — idempotent best-effort destroy.
# gcr_vm_destroy SERVER_ID — success means DELETE returned HTTP 2xx.
gcr_vm_destroy() {
if ! gcr_hcloud_req DELETE "/servers/$1"; then
gcr_log warn --ns=hcloud "destroy failed or already gone: server $1"
@@ -225,6 +327,77 @@ gcr_vm_destroy() {
fi
}
# Only cleanup records establish prior ownership, making DELETE 404 a
# confirmed-absent success rather than an ambiguous lookup failure.
gcr_vm_destroy_owned() {
gcr_vm_destroy "$1" && return 0
[ "${GCR_LAST_HTTP:-}" = "404" ]
}
# Caller holds the lifecycle path's existing ownership locks.
gcr_vm_cleanup_pending() {
cleanup_job="$1"; cleanup_attempt="$2"; cleanup_rec="$3"
cleanup_vm_id="$(gcr_record_field "$cleanup_rec" vm_id)"
if [ "$(gcr_record_field "$cleanup_rec" cleanup_vm_destroyed)" != "true" ]; then
gcr_vm_destroy_owned "$cleanup_vm_id" || return 1
cleanup_rec="$(printf '%s' "$cleanup_rec" | jq -c '.cleanup_vm_destroyed = true')"
gcr_record_put "$cleanup_job" "$cleanup_attempt" "$cleanup_rec" || return 1
fi
if [ "$(gcr_record_field "$cleanup_rec" cleanup_refund_budget)" = "true" ] \
&& [ "$(gcr_record_field "$cleanup_rec" cleanup_budget_released)" != "true" ]; then
cleanup_rate="$(gcr_record_field "$cleanup_rec" budget_rate)"
cleanup_ttl="$(gcr_record_field "$cleanup_rec" ttl_min)"
cleanup_refund_key="$(gcr_alloc_key "$cleanup_job" "$cleanup_attempt")"
gcr_budget_refund_once "$cleanup_refund_key" "$cleanup_rate" "$cleanup_ttl" \
|| return 1
cleanup_rec="$(printf '%s' "$cleanup_rec" | jq -c '.cleanup_budget_released = true')"
gcr_record_put "$cleanup_job" "$cleanup_attempt" "$cleanup_rec" || return 1
fi
gcr_record_del "$cleanup_job" "$cleanup_attempt"
}
# Persist intent before DELETE. Normal lifecycle teardown never changes budget;
# failed creation passes REFUND_BUDGET=true to release its unused reservation.
gcr_vm_cleanup_start() {
cleanup_job="$1"; cleanup_attempt="$2"; cleanup_source="$3"
cleanup_reason="$4"; cleanup_refund="$5"
cleanup_rec="$(printf '%s' "$cleanup_source" | jq -c \
--arg reason "$cleanup_reason" --argjson refund "$cleanup_refund" \
'.status = "cleanup_pending"
| .cleanup_reason = $reason
| .cleanup_refund_budget = $refund
| .cleanup_vm_destroyed = false
| .cleanup_budget_released = false')"
gcr_record_put "$cleanup_job" "$cleanup_attempt" "$cleanup_rec" || return 1
gcr_vm_cleanup_pending "$cleanup_job" "$cleanup_attempt" "$cleanup_rec"
}
# Caller holds allocation and admission locks. A failed primary write first
# persists cleanup ownership; reservation is released only after destroy.
gcr_vm_record_created() {
record_job="$1"; record_attempt="$2"; record_repo="$3"; record_label="$4"
record_created="$5"; record_vm_id="$6"; record_vm_name="$7"
record_ttl="$8"; record_type="$9"; shift 9; record_rate="$1"
record_rec="$(jq -n --arg j "$record_job" --arg a "$record_attempt" \
--arg r "$record_repo" --arg l "$record_label" --arg t "$record_created" \
--arg v "$record_vm_id" --arg vn "$record_vm_name" --arg ttl "$record_ttl" \
--arg st "$record_type" --arg rate "$record_rate" \
'{job_id:$j, run_attempt:$a, repo:$r, label:$l,
created_at:$t, ttl_min:($ttl|tonumber), vm_id:($v|tonumber),
vm_name:$vn, server_type:$st, budget_rate:$rate,
bootstrapped:false, status:"pending_vm"}')"
gcr_record_put "$record_job" "$record_attempt" "$record_rec" && return 0
if ! gcr_vm_cleanup_start "$record_job" "$record_attempt" "$record_rec" \
state-write-failed true; then
cleanup_rec="$(gcr_record_get "$record_job" "$record_attempt")"
[ "$(gcr_record_field "$cleanup_rec" status)" = "cleanup_pending" ] && return 1
gcr_log error --ns=alloc "cannot persist cleanup record job=$record_job vm=$record_vm_id"
return 1
fi
return 1
}
gcr_vm_public_ip() {
# gcr_vm_public_ip SERVER_ID -> ipv4 or empty
if gcr_hcloud_req GET "/servers/$1"; then
package/gitea-runner-controller/state.sh
+54 -3
View File
@@ -257,6 +257,21 @@ gcr_budget_can_add() {
return 0
}
# Atomic replacement preserves last valid total when a write fails.
gcr_budget_write() {
gcr_budget_file="$1"; gcr_budget_value="$2"
gcr_budget_tmp="$(mktemp "$(dirname "$gcr_budget_file")/.budget.XXXXXX")" \
|| return 1
if ! printf '%s\n' "$gcr_budget_value" > "$gcr_budget_tmp"; then
rm -f "$gcr_budget_tmp"
return 1
fi
if ! mv -f "$gcr_budget_tmp" "$gcr_budget_file"; then
rm -f "$gcr_budget_tmp"
return 1
fi
}
# Caller holds admission lock and has already checked gcr_budget_can_add.
gcr_budget_add() {
rate="$1"; ttl_min="$2"
@@ -264,10 +279,46 @@ gcr_budget_add() {
file="$GCR_STATE_DIR/budget/$month"
current="$(cat "$file" 2>/dev/null || echo 0)"
projected="$(awk -v c="$current" -v r="$rate" -v t="$ttl_min" 'BEGIN {printf "%.4f", c + r * t / 60}')"
printf '%s\n' "$projected" > "$file"
return 0
gcr_budget_write "$file" "$projected"
}
# Caller holds admission lock and is rolling back a matching budget addition.
gcr_budget_sub() {
rate="$1"; ttl_min="$2"
month="$(date -u '+%Y-%m')"
file="$GCR_STATE_DIR/budget/$month"
current="$(cat "$file" 2>/dev/null || echo 0)"
projected="$(awk -v c="$current" -v r="$rate" -v t="$ttl_min" \
'BEGIN {v = c - r * t / 60; if (v < 0) v = 0; printf "%.4f", v}')"
gcr_budget_write "$file" "$projected"
}
# Claim is durable before credit. Existing claim means credit is consumed:
# it may have completed, or it may have leaked fail-closed after a crash.
# Never subtract twice when outcome between aggregate and state writes is unknown.
gcr_budget_refund_once() {
refund_key="$1"; refund_rate="$2"; refund_ttl="$3"
refund_month="$(date -u '+%Y-%m')"
refund_root="$GCR_STATE_DIR/budget/refunds/$refund_month"
refund_claim="$refund_root/$refund_key"
mkdir -p "$refund_root" || return 1
if ! mkdir "$refund_claim" 2>/dev/null; then
if [ -d "$refund_claim" ]; then
if [ ! -f "$refund_claim/status" ]; then
gcr_log error --ns=budget \
"refund outcome uncertain key=$refund_key; retaining fail-closed claim"
fi
return 0
fi
return 1
fi
if ! printf '%s %s\n' "$refund_rate" "$refund_ttl" > "$refund_claim/intent"; then
return 1
fi
gcr_budget_sub "$refund_rate" "$refund_ttl" || return 1
printf 'refunded\n' > "$refund_claim/status" || return 1
}
gcr_active_records() {
grep -El '"status"[[:space:]]*:[[:space:]]*"(pending_vm|vm_active|idle_vm|deferred)"' "$GCR_STATE_DIR"/jobs/*.json 2>/dev/null || true
grep -El '"status"[[:space:]]*:[[:space:]]*"(pending_vm|vm_active|idle_vm|deferred|cleanup_pending|create_ambiguous)"' "$GCR_STATE_DIR"/jobs/*.json 2>/dev/null || true
}
package/gitea-runner-controller/webhook.sh
+35 -34
View File
@@ -116,7 +116,7 @@ gcr_alloc() {
fi
set -- $profile
server_type="$1"; ttl_min="$2"; rate="$3"
server_type="$1"; ttl_min="$2"
if ! gcr_lock_acquire admission; then
rec="$(jq -n --arg j "$job_id" --arg a "$attempt" --arg r "$repo" \
@@ -183,15 +183,6 @@ gcr_alloc() {
return 0
fi
if ! gcr_budget_can_add "$rate" "$ttl_min"; then
gcr_record_del "$job_id" "$attempt"
gcr_lock_release admission
gcr_lock_release "$key"
gcr_event "budget-refused" "$job_id" "{\"rate\":$rate,\"ttl_min\":$ttl_min}"
RESPONSE_CODE=202; RESPONSE_BODY="refused: monthly budget exhausted"
return 0
fi
reg_token="$(gcr_gitea_registration_token "$repo")" || {
gcr_record_del "$job_id" "$attempt"
gcr_lock_release admission
@@ -203,26 +194,27 @@ gcr_alloc() {
vm_name="gcr-${job_id}-${attempt}"
created_at="$(gcr_now_epoch)"
vm_id="$(gcr_vm_create "$vm_name" "$label" "$server_type" "$ttl_min" \
"$reg_token" "$job_id" "$attempt" "$repo")" || {
gcr_record_del "$job_id" "$attempt"
create_status=0
created="$(gcr_vm_create "$vm_name" "$label" "$server_type" "$ttl_min" \
"$reg_token" "$job_id" "$attempt" "$repo")" || create_status="$?"
if [ "$create_status" -ne 0 ]; then
if [ "$create_status" -eq 2 ]; then
gcr_event "vm-create-ambiguous" "$job_id" "{}"
RESPONSE_CODE=202; RESPONSE_BODY="VM creation pending recovery"
else
gcr_record_del "$job_id" "$attempt"
gcr_event "vm-create-failed" "$job_id" "{}"
RESPONSE_CODE=202; RESPONSE_BODY="VM creation failed"
fi
gcr_lock_release admission
gcr_lock_release "$key"
gcr_event "vm-create-failed" "$job_id" "{}"
RESPONSE_CODE=202; RESPONSE_BODY="VM creation failed"
return 0
}
gcr_budget_add "$rate" "$ttl_min"
rec="$(jq -n --arg j "$job_id" --arg a "$attempt" --arg r "$repo" \
--arg l "$label" --arg t "$created_at" --arg v "$vm_id" \
--arg vn "$vm_name" --arg ttl "$ttl_min" \
'{job_id:$j, run_attempt:$a, repo:$r, label:$l,
created_at:$t, ttl_min:($ttl|tonumber), vm_id:($v|tonumber),
vm_name:$vn, bootstrapped:false, status:"pending_vm"}')"
if ! gcr_record_put "$job_id" "$attempt" "$rec"; then
gcr_vm_destroy "$vm_id" || true
fi
set -- $created
vm_id="$1"; actual_server_type="$2"; actual_rate="$3"
if ! gcr_vm_record_created "$job_id" "$attempt" "$repo" "$label" \
"$created_at" "$vm_id" "$vm_name" "$ttl_min" \
"$actual_server_type" "$actual_rate"; then
gcr_lock_release admission
gcr_lock_release "$key"
RESPONSE_CODE=202; RESPONSE_BODY="VM state write failed"
@@ -266,10 +258,14 @@ gcr_deallocate() {
if ! gcr_gitea_runner_disabled "$(gcr_record_field "$rec" repo)" "$runner_name" true \
|| ! gcr_vm_runner_service "$vm_id" stop; then
gcr_lock_release idle-pool
gcr_vm_destroy "$vm_id" || true
gcr_record_del "$job_id" "$attempt"
if gcr_vm_cleanup_start "$job_id" "$attempt" "$rec" idle-stop-failed false; then
gcr_event "vm-destroyed" "$job_id" \
"{\"vm_id\":$vm_id,\"reason\":\"idle-stop-failed\"}"
else
gcr_event "vm-cleanup-pending" "$job_id" \
"{\"vm_id\":$vm_id,\"reason\":\"idle-stop-failed\"}"
fi
gcr_lock_release "$key"
gcr_event "vm-destroyed" "$job_id" "{\"vm_id\":$vm_id,\"reason\":\"idle-stop-failed\"}"
return 0
fi
gcr_record_put "$job_id" "$attempt" "$idle_rec"
@@ -288,11 +284,16 @@ gcr_deallocate() {
gcr_vm_collect_diagnostics "$vm_id" "$ip" "$job_id" "$new_status" || true
;;
esac
gcr_vm_destroy "$vm_id" || true
gcr_event "vm-destroyed" "$job_id" "{\"vm_id\":$vm_id,\"reason\":\"$new_status\"}"
if gcr_vm_cleanup_start "$job_id" "$attempt" "$rec" "$new_status" false; then
gcr_event "vm-destroyed" "$job_id" \
"{\"vm_id\":$vm_id,\"reason\":\"$new_status\"}"
else
gcr_event "vm-cleanup-pending" "$job_id" \
"{\"vm_id\":$vm_id,\"reason\":\"$new_status\"}"
fi
else
gcr_record_del "$job_id" "$attempt"
fi
gcr_record_del "$job_id" "$attempt"
gcr_lock_release "$key"
}
test/package/gitea-runner-controller/test/highmem-label.sh
+242 -13
View File
@@ -5,6 +5,9 @@ set -eu
. "$STATE_SH"
. "$DECIDE_SH"
. "$HCLOUD_SH"
. "$GITEA_SH"
. "$CONTROLLER_SH"
. "$WEBHOOK_SH"
gcr_state_init
export GCR_ALLOWED_REPOS='hectic-lab/util.nix'
@@ -13,10 +16,22 @@ export GCR_NIX_IMAGE_ID='424242'
test "$(gcr_label_ttl ubuntu-latest)" = '180'
test "$(gcr_label_ttl nix)" = '480'
test "$(gcr_decide ubuntu-latest hectic-lab/util.nix)" = 'cx53 180 0.032'
test "$(gcr_decide nix hectic-lab/util.nix)" = 'cx53 480 0.032'
test "$(gcr_label_candidates ubuntu-latest | head -n1)" = 'cx53 nbg1 amd64'
test "$(gcr_label_candidates nix | head -n1)" = 'cx53 nbg1 amd64'
test "$(gcr_decide ubuntu-latest hectic-lab/util.nix)" = 'cx23 180 0.004'
test "$(gcr_decide nix hectic-lab/util.nix)" = 'cx23 480 0.004'
standard_candidates='cx23 nbg1 amd64
cx23 fsn1 amd64
cx23 hel1 amd64
cx33 nbg1 amd64
cx33 fsn1 amd64
cx33 hel1 amd64
cx43 nbg1 amd64
cx43 fsn1 amd64
cx43 hel1 amd64
cx53 nbg1 amd64
cx53 fsn1 amd64
cx53 hel1 amd64'
test "$(gcr_label_candidates ubuntu-latest)" = "$standard_candidates"
test "$(gcr_label_candidates nix)" = "$standard_candidates"
test "$(gcr_image_id_for_arch amd64 ubuntu-latest)" = '313131'
test "$(gcr_image_id_for_arch amd64 nix)" = '424242'
@@ -44,29 +59,36 @@ if gcr_budget_can_add 0.8550 480; then
printf 'highmem full-TTL reservation must obey budget cap\n' >&2
exit 1
fi
export GCR_BUDGET_EUR_MONTHLY='6.84'
gcr_budget_add 0.8550 480
test "$(cat "$GCR_STATE_DIR/budget/$(date -u '+%Y-%m')")" = '6.8400'
if gcr_budget_write "$GCR_STATE_DIR/missing/budget" 1 2>/dev/null; then
printf 'budget writes must propagate failures\n' >&2
exit 1
fi
rm -f "$GCR_STATE_DIR/budget/$(date -u '+%Y-%m')"
export GCR_BUDGET_EUR_MONTHLY='10'
calls="$GCR_STATE_DIR/hcloud-calls"
sleep() { :; }
gcr_hcloud_token() { printf token; }
gcr_hcloud_req() {
method="$1"; path="$2"; body="${3:-}"
test "$method" = POST
GCR_LAST_BODY="$GCR_STATE_DIR/last-body.json"
if [ "$method" = GET ]; then
printf '{"servers":[]}\n' > "$GCR_LAST_BODY"
GCR_LAST_HTTP=200
return 0
fi
test "$path" = /servers
printf '%s\n' "$body" | jq -c . >> "$calls"
count="$(wc -l < "$calls" | tr -d ' ')"
GCR_LAST_BODY="$GCR_STATE_DIR/last-body.json"
case "$count" in
1|2) return 1 ;;
3) printf '{"server":{"id":9001}}\n' > "$GCR_LAST_BODY"; return 0 ;;
1|2) GCR_LAST_HTTP=412; return 1 ;;
3) GCR_LAST_HTTP=201; printf '{"server":{"id":9001}}\n' > "$GCR_LAST_BODY"; return 0 ;;
*) return 1 ;;
esac
}
vm_id="$(gcr_vm_create gcr-9001-1 gross-nix-x86-highmem ccx53 480 reg-token 9001 1 hectic-lab/util.nix)"
test "$vm_id" = '9001'
created="$(gcr_vm_create gcr-9001-1 gross-nix-x86-highmem ccx53 480 reg-token 9001 1 hectic-lab/util.nix)"
test "$created" = '9001 ccx53 0.8550'
test "$(wc -l < "$calls" | tr -d ' ')" = '3'
jq -e 'select(.server_type == "ccx53" and .location == "nbg1" and .labels["gcr.arch"] == "amd64" and .labels["gcr.label"] == "gross-nix-x86-highmem" and .labels["gcr.ttl-min"] == "480")' "$calls" >/dev/null
jq -e 'select(.server_type == "ccx53" and .location == "fsn1" and .labels["gcr.arch"] == "amd64")' "$calls" >/dev/null
@@ -75,3 +97,210 @@ if jq -e 'select(.server_type != "ccx53" or .labels["gcr.arch"] != "amd64")' "$c
printf 'highmem VM creation attempted non-ccx53 or non-amd64 candidate\n' >&2
exit 1
fi
# Standard allocation charges actual fallback type after cx23 capacity failures.
rm -f "$calls" "$GCR_STATE_DIR/budget/$(date -u '+%Y-%m')"
export GCR_BUDGET_EUR_MONTHLY='10'
gcr_gitea_registration_token() { printf token; }
gcr_hcloud_req() {
method="$1"; path="$2"; body="${3:-}"
GCR_LAST_BODY="$GCR_STATE_DIR/last-body.json"
if [ "$method" = GET ]; then
printf 'lookup\n' >> "$GCR_STATE_DIR/lookups"
printf '{"servers":[]}\n' > "$GCR_LAST_BODY"
GCR_LAST_HTTP=200
return 0
fi
test "$path" = /servers
printf '%s\n' "$body" | jq -c . >> "$calls"
count="$(wc -l < "$calls" | tr -d ' ')"
case "$count" in
1|2|3) GCR_LAST_HTTP=412; return 1 ;;
4) GCR_LAST_HTTP=201; printf '{"server":{"id":9002}}\n' > "$GCR_LAST_BODY"; return 0 ;;
*) return 1 ;;
esac
}
gcr_alloc 9002 1 hectic-lab/util.nix '["nix"]'
test "$RESPONSE_CODE" = 202
test "$RESPONSE_BODY" = 'allocated gcr-9002-1'
test "$(wc -l < "$calls" | tr -d ' ')" = '4'
test "$(wc -l < "$GCR_STATE_DIR/lookups" | tr -d ' ')" = '3'
jq -e -s 'map(.server_type) == ["cx23", "cx23", "cx23", "cx33"]' "$calls" >/dev/null
test "$(gcr_server_hourly_rate cx33)" = '0.008'
test "$(cat "$GCR_STATE_DIR/budget/$(date -u '+%Y-%m')")" = '0.0640'
# Ambiguous 5xx response adopts only exact deterministic managed identity.
gcr_record_del 9002 1
rm -f "$calls" "$GCR_STATE_DIR/budget/$(date -u '+%Y-%m')"
gcr_hcloud_req() {
method="$1"; body="${3:-}"
GCR_LAST_BODY="$GCR_STATE_DIR/last-body.json"
if [ "$method" = POST ]; then
printf '%s\n' "$body" | jq -c . >> "$calls"
GCR_LAST_HTTP=000
return 1
fi
GCR_LAST_HTTP=200
printf '%s\n' '{"servers":[{"id":9010,"name":"gcr-9010-1","server_type":{"name":"cx23"},"labels":{"gitea-runner-controller":"managed","gcr.job-id":"9010","gcr.run-attempt":"1","gcr.label":"nix","gcr.location":"nbg1","gcr.arch":"amd64"}}]}' > "$GCR_LAST_BODY"
}
gcr_alloc 9010 1 hectic-lab/util.nix '["nix"]'
test "$RESPONSE_BODY" = 'allocated gcr-9010-1'
test "$(wc -l < "$calls" | tr -d ' ')" = 1
found_rec="$(gcr_record_get 9010 1)"
test "$(gcr_record_field "$found_rec" status)" = pending_vm
test "$(gcr_record_field "$found_rec" vm_id)" = 9010
test "$(cat "$GCR_STATE_DIR/budget/$(date -u '+%Y-%m')")" = '0.0320'
# Ambiguous POST plus failed lookup retains reservation and blocks fallback;
# reconciler adopts a later exact identity without deleting or refunding it.
gcr_record_del 9010 1
rm -f "$calls" "$GCR_STATE_DIR/budget/$(date -u '+%Y-%m')" "$GCR_STATE_DIR/lookups"
gcr_hcloud_req() {
method="$1"; body="${3:-}"
if [ "$method" = POST ]; then
printf '%s\n' "$body" | jq -c . >> "$calls"
GCR_LAST_HTTP=500
return 1
fi
printf 'lookup-failed\n' >> "$GCR_STATE_DIR/lookups"
GCR_LAST_HTTP=503
return 1
}
gcr_alloc 9012 1 hectic-lab/util.nix '["nix"]'
test "$RESPONSE_BODY" = 'VM creation pending recovery'
test "$(wc -l < "$calls" | tr -d ' ')" = 1
ambiguous_rec="$(gcr_record_get 9012 1)"
test "$(gcr_record_field "$ambiguous_rec" status)" = create_ambiguous
test "$(gcr_record_field "$ambiguous_rec" budget_rate)" = 0.004
test "$(gcr_count_active)" = 1
test "$(cat "$GCR_STATE_DIR/budget/$(date -u '+%Y-%m')")" = '0.0320'
gcr_sweep_create_ambiguous
test "$(gcr_record_field "$(gcr_record_get 9012 1)" status)" = create_ambiguous
test "$(wc -l < "$calls" | tr -d ' ')" = 1
gcr_hcloud_req() {
test "$1" = GET
GCR_LAST_BODY="$GCR_STATE_DIR/last-body.json"
GCR_LAST_HTTP=200
printf '%s\n' '{"servers":[{"id":9012,"name":"gcr-9012-1","server_type":{"name":"cx23"},"labels":{"gitea-runner-controller":"managed","gcr.job-id":"9012","gcr.run-attempt":"1","gcr.label":"nix","gcr.location":"nbg1","gcr.arch":"amd64"}}]}' > "$GCR_LAST_BODY"
}
gcr_vm_destroy() { printf 'unexpected-destroy %s\n' "$1" >> "$GCR_STATE_DIR/ambiguity-destroys"; return 1; }
gcr_sweep_create_ambiguous
recovered_rec="$(gcr_record_get 9012 1)"
test "$(gcr_record_field "$recovered_rec" status)" = pending_vm
test "$(gcr_record_field "$recovered_rec" vm_id)" = 9012
test "$(gcr_record_field "$recovered_rec" server_type)" = cx23
test "$(gcr_record_field "$recovered_rec" budget_rate)" = 0.004
test "$(cat "$GCR_STATE_DIR/budget/$(date -u '+%Y-%m')")" = '0.0320'
test ! -e "$GCR_STATE_DIR/ambiguity-destroys"
gcr_record_del 9012 1
rm -f "$GCR_STATE_DIR/budget/$(date -u '+%Y-%m')"
# Fallback candidates exceeding remaining budget never reach Hetzner.
rm -f "$calls" "$GCR_STATE_DIR/budget/$(date -u '+%Y-%m')"
export GCR_BUDGET_EUR_MONTHLY='0.05'
gcr_hcloud_req() {
method="$1"; path="$2"; body="${3:-}"
GCR_LAST_BODY="$GCR_STATE_DIR/last-body.json"
if [ "$method" = GET ]; then
printf '{"servers":[]}\n' > "$GCR_LAST_BODY"
GCR_LAST_HTTP=200
return 0
fi
test "$path" = /servers
printf '%s\n' "$body" | jq -c . >> "$calls"
GCR_LAST_HTTP=412
return 1
}
gcr_alloc 9003 1 hectic-lab/util.nix '["nix"]'
test "$RESPONSE_BODY" = 'VM creation failed'
test "$(wc -l < "$calls" | tr -d ' ')" = '3'
jq -e -s 'all(.server_type == "cx23")' "$calls" >/dev/null
test "$(cat "$GCR_STATE_DIR/budget/$(date -u '+%Y-%m')")" = '0.0000'
# Deferred allocation keeps cleanup ownership and reservation across failed
# DELETE, failed post-DELETE state update, and cleanup-owned 404 retry.
rm -f "$calls" "$GCR_STATE_DIR/budget/$(date -u '+%Y-%m')"
export GCR_BUDGET_EUR_MONTHLY='10'
deferred='{"job_id":"9005","run_attempt":"1","repo":"hectic-lab/util.nix","label":"nix","created_at":"0","ttl_min":null,"vm_id":"","vm_name":"","status":"deferred"}'
printf '%s\n' "$deferred" > "$GCR_STATE_DIR/jobs/9005-1.json"
gcr_gitea_job_state() { printf queued; }
record_put_count="$GCR_STATE_DIR/record-put-count"
gcr_record_put() {
count="$(cat "$record_put_count" 2>/dev/null || echo 0)"
count=$((count + 1))
printf '%s\n' "$count" > "$record_put_count"
case "$count" in 1|3|5|8) return 1 ;; esac
tmp="$(mktemp "$(dirname "$(gcr_record_path "$1" "$2")")/.tmp.XXXXXX")"
printf '%s\n' "$3" > "$tmp"
mv -f "$tmp" "$(gcr_record_path "$1" "$2")"
}
gcr_hcloud_req() {
test "$1" = POST
printf '%s\n' "$3" | jq -c . >> "$calls"
GCR_LAST_BODY="$GCR_STATE_DIR/last-body.json"
printf '{"server":{"id":9005}}\n' > "$GCR_LAST_BODY"
}
gcr_vm_destroy() {
printf 'destroy-failed %s\n' "$1" >> "$GCR_STATE_DIR/destroy-calls"
return 1
}
gcr_alloc_deferred 9005 1
cleanup_rec="$(gcr_record_get 9005 1)"
test "$(gcr_record_field "$cleanup_rec" status)" = cleanup_pending
test "$(gcr_record_field "$cleanup_rec" vm_id)" = 9005
test "$(gcr_record_field "$cleanup_rec" budget_rate)" = 0.004
test "$(cat "$GCR_STATE_DIR/budget/$(date -u '+%Y-%m')")" = '0.0320'
gcr_vm_destroy() { printf 'deleted %s\n' "$1" >> "$GCR_STATE_DIR/destroy-calls"; }
gcr_sweep_cleanup_pending
cleanup_rec="$(gcr_record_get 9005 1)"
test "$(gcr_record_field "$cleanup_rec" status)" = cleanup_pending
test "$(gcr_record_field "$cleanup_rec" cleanup_vm_destroyed)" = false
test "$(cat "$GCR_STATE_DIR/budget/$(date -u '+%Y-%m')")" = '0.0320'
gcr_vm_destroy() {
printf 'absent %s\n' "$1" >> "$GCR_STATE_DIR/destroy-calls"
GCR_LAST_HTTP=404
return 1
}
gcr_sweep_cleanup_pending
cleanup_rec="$(gcr_record_get 9005 1)"
test "$(gcr_record_field "$cleanup_rec" cleanup_vm_destroyed)" = true
test "$(cat "$GCR_STATE_DIR/budget/$(date -u '+%Y-%m')")" = '0.0000'
gcr_sweep_cleanup_pending
test ! -e "$(gcr_record_path 9005 1)"
test "$(cat "$GCR_STATE_DIR/budget/$(date -u '+%Y-%m')")" = '0.0000'
grep -q '^destroy-failed 9005$' "$GCR_STATE_DIR/destroy-calls"
grep -q '^deleted 9005$' "$GCR_STATE_DIR/destroy-calls"
grep -q '^absent 9005$' "$GCR_STATE_DIR/destroy-calls"
test "$(grep -c '^absent 9005$' "$GCR_STATE_DIR/destroy-calls")" = 1
# Confirmed ambiguous absence uses same monotonic refund claim. Failed state
# persistence after credit cannot make retry subtract reservation twice.
gcr_budget_add 0.004 480
ambiguous_absent='{"job_id":"9013","run_attempt":"1","repo":"hectic-lab/util.nix","label":"nix","created_at":"0","ttl_min":480,"vm_id":"","vm_name":"gcr-9013-1","server_type":"cx23","budget_rate":"0.004","candidate_location":"nbg1","candidate_arch":"amd64","bootstrapped":false,"status":"create_ambiguous"}'
printf '%s\n' "$ambiguous_absent" > "$GCR_STATE_DIR/jobs/9013-1.json"
gcr_hcloud_req() {
test "$1" = GET
GCR_LAST_BODY="$GCR_STATE_DIR/last-body.json"
GCR_LAST_HTTP=200
printf '{"servers":[]}\n' > "$GCR_LAST_BODY"
}
gcr_sweep_create_ambiguous
absence_rec="$(gcr_record_get 9013 1)"
test "$(gcr_record_field "$absence_rec" status)" = cleanup_pending
test "$(gcr_record_field "$absence_rec" cleanup_vm_destroyed)" = true
test "$(cat "$GCR_STATE_DIR/budget/$(date -u '+%Y-%m')")" = '0.0000'
gcr_sweep_cleanup_pending
test ! -e "$(gcr_record_path 9013 1)"
test "$(cat "$GCR_STATE_DIR/budget/$(date -u '+%Y-%m')")" = '0.0000'
# Reservation write failure aborts before any Hetzner request.
rm -f "$calls"
rm -rf "$GCR_STATE_DIR/budget"
printf 'not-a-directory\n' > "$GCR_STATE_DIR/budget"
gcr_alloc 9006 1 hectic-lab/util.nix '["nix"]'
test "$RESPONSE_BODY" = 'VM creation failed'
test ! -e "$calls"
test ! -e "$(gcr_record_path 9006 1)"
test/package/gitea-runner-controller/test/idle-reuse.sh
+8 -2
View File
@@ -19,7 +19,11 @@ gcr_now_epoch() { printf '%s' "$NOW"; }
calls="$GCR_STATE_DIR/calls"
gcr_budget_add() { printf 'budget\n' >> "$calls"; }
gcr_gitea_registration_token() { printf 'token'; printf 'token\n' >> "$calls"; }
gcr_vm_create() { printf 'create\n' >> "$calls"; printf '99'; }
gcr_vm_create() {
printf 'create\n' >> "$calls"
gcr_budget_add 0.032 180
printf '99 cx53 0.032'
}
gcr_vm_destroy() { printf 'destroy vm=%s\n' "$1" >> "$calls"; }
gcr_vm_runner_service() { printf 'runner %s vm=%s\n' "$2" "$1" >> "$calls"; }
gcr_gitea_runner_disabled() { printf 'runner-disabled %s %s\n' "$2" "$3" >> "$calls"; }
@@ -270,6 +274,7 @@ set -eu
. "$LOG_SH"
. "$STATE_SH"
. "$DECIDE_SH"
. "$HCLOUD_SH"
. "$WEBHOOK_SH"
gcr_now_epoch() { printf '2000'; }
gcr_budget_add() { printf 'budget %s\n' "$1" >> "$GCR_STATE_DIR/admission-calls"; }
@@ -277,7 +282,8 @@ gcr_gitea_registration_token() { printf token; }
gcr_vm_create() {
sleep 1
printf 'create %s\n' "$6" >> "$GCR_STATE_DIR/admission-calls"
printf '%s' "$6"
gcr_budget_add 0.032 180
printf '%s cx53 0.032' "$6"
}
gcr_vm_destroy() { :; }
gcr_state_init
test/package/gitea-runner-controller/test/reap-diagnostics.sh
+27
View File
@@ -63,6 +63,29 @@ gcr_reap_finished_jobs
test "$(gcr_record_get 101 1)" = "$idle_once"
test ! -e "$(gcr_record_path 102 1)"
# Reaper keeps cleanup ownership after DELETE failure and retries next sweep.
record_delete_fail='{"job_id":"104","run_attempt":"1","repo":"hinterland/hearth","label":"nix","created_at":"1","ttl_min":480,"vm_id":44,"vm_name":"gcr-104-1","bootstrapped":true,"status":"vm_active"}'
gcr_record_put 104 1 "$record_delete_fail"
gcr_gitea_job_state() {
case "$2" in
104) printf 'completed:failure' ;;
*) return 1 ;;
esac
}
gcr_vm_destroy() {
printf 'destroy-failed vm=%s\n' "$1" >> "$calls"
return 1
}
gcr_reap_finished_jobs
test "$(gcr_record_field "$(gcr_record_get 104 1)" status)" = cleanup_pending
test "$(gcr_count_active)" = 1
gcr_vm_destroy() { printf 'destroy-retry vm=%s\n' "$1" >> "$calls"; }
gcr_sweep_cleanup_pending
test ! -e "$(gcr_record_path 104 1)"
grep -q '^destroy-failed vm=44$' "$calls"
grep -q '^destroy-retry vm=44$' "$calls"
calls_ip_fail="$GCR_STATE_DIR/calls-ip-fail"
calls="$calls_ip_fail"
record_ip_fail='{"job_id":"103","run_attempt":"1","repo":"hinterland/hearth","label":"gross-nix-x86-perf","created_at":"1","ttl_min":480,"vm_id":43,"vm_name":"gcr-103-1","bootstrapped":true,"status":"vm_active"}'
@@ -79,6 +102,10 @@ gcr_vm_public_ip() {
return 1
}
gcr_vm_destroy() {
printf 'destroy vm=%s\n' "$1" >> "$calls"
}
gcr_reap_finished_jobs
grep -q 'diag vm=43 ip= job=103 reason=completed:failure' "$calls_ip_fail"
test/package/gitea-runner-controller/test/webhook-diagnostics.sh
+23
View File
@@ -71,6 +71,29 @@ grep -q 'diag vm=53 ip= job=203 reason=completed:failure' "$calls_ip_fail"
grep -q 'destroy vm=53' "$calls_ip_fail"
test ! -e "$(gcr_record_path 203 1)"
# Terminal webhook retains ownership and capacity until DELETE succeeds.
record_delete_fail='{"job_id":"204","run_attempt":"1","repo":"hinterland/hearth","label":"nix","created_at":"1","ttl_min":480,"vm_id":54,"vm_name":"gcr-204-1","bootstrapped":true,"status":"vm_active"}'
gcr_record_put 204 1 "$record_delete_fail"
gcr_vm_destroy() {
printf 'destroy-failed vm=%s\n' "$1" >> "$calls_ip_fail"
return 1
}
gcr_deallocate 204 1 completed:failure
cleanup_rec="$(gcr_record_get 204 1)"
test "$(gcr_record_field "$cleanup_rec" status)" = cleanup_pending
test "$(gcr_count_active)" = 1
test "$(gcr_count_active_repo hinterland/hearth)" = 1
gcr_vm_destroy() { printf 'destroy-retry vm=%s\n' "$1" >> "$calls_ip_fail"; }
key="$(gcr_alloc_key 204 1)"
gcr_lock_acquire "$key"
gcr_vm_cleanup_pending 204 1 "$(gcr_record_get 204 1)"
gcr_lock_release "$key"
test ! -e "$(gcr_record_path 204 1)"
test "$(gcr_count_active)" = 0
grep -q '^destroy-failed vm=54$' "$calls_ip_fail"
grep -q '^destroy-retry vm=54$' "$calls_ip_fail"
# Gitea emits zero-based run_attempt values for initial workflow jobs.
gcr_read_request() {
gcr_hdr_event_type=workflow_job