hinterland/hearth
2
2
Fork 1
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix: runners
runner nix smoke / nix label and flake smoke (push) Failing after 1m1s
Details

Browse Source
This commit is contained in:
yukkop
2026-09-11 05:58:04 +00:00
parent f0317efb6e
commit 4a88e7c6e8
9 changed files with 736 additions and 99 deletions
Download Patch File Download Diff File Expand all files Collapse all files
package/gitea-runner-controller/controller.sh
+150 -37
View File
@@ -40,10 +40,16 @@ gcr_sweep_ttl() {
fi
gcr_log info --ns=sweep "idle slot expired job=$job_id vm=$vm_id"
if [ -n "$vm_id" ] && [ "$vm_id" != "null" ] && [ "$vm_id" != "0" ]; then
gcr_vm_destroy "$vm_id" || true
gcr_event "vm-destroyed" "$job_id" "{\"vm_id\":$vm_id,\"reason\":\"idle-expired\"}"
if gcr_vm_cleanup_start "$job_id" "$attempt" "$rec" idle-expired false; then
gcr_event "vm-destroyed" "$job_id" \
"{\"vm_id\":$vm_id,\"reason\":\"idle-expired\"}"
else
gcr_event "vm-cleanup-pending" "$job_id" \
"{\"vm_id\":$vm_id,\"reason\":\"idle-expired\"}"
fi
else
gcr_record_del "$job_id" "$attempt"
fi
gcr_record_del "$job_id" "$attempt"
gcr_lock_release idle-pool
continue
fi
@@ -78,11 +84,17 @@ gcr_sweep_ttl() {
if [ -n "$vm_id" ] && [ "$vm_id" != "null" ] && [ "$vm_id" != "0" ]; then
ip="$(gcr_vm_public_ip "$vm_id" || true)"
gcr_vm_collect_diagnostics "$vm_id" "$ip" "$job_id" ttl || true
gcr_vm_destroy "$vm_id" || true
gcr_event "vm-destroyed" "$job_id" "{\"vm_id\":$vm_id,\"reason\":\"ttl\"}"
if gcr_vm_cleanup_start "$job_id" "$attempt" "$rec" ttl false; then
gcr_event "vm-destroyed" "$job_id" \
"{\"vm_id\":$vm_id,\"reason\":\"ttl\"}"
else
gcr_event "vm-cleanup-pending" "$job_id" \
"{\"vm_id\":$vm_id,\"reason\":\"ttl\"}"
fi
else
gcr_record_del "$job_id" "$attempt"
fi
gcr_event "job-ttl-expired" "$job_id" "{\"age\":$age}"
gcr_record_del "$job_id" "$attempt"
gcr_lock_release "$key"
fi
done
@@ -108,8 +120,20 @@ gcr_sweep_orphan_vms() {
if ! gcr_record_exists_for_vm_id "$vm_id"; then
gcr_log warn --ns=sweep "orphan VM $vm_id job=$jid attempt=$att -> destroy"
gcr_vm_destroy "$vm_id" || true
gcr_event "orphan-vm-destroyed" "${jid:-unknown}" "{\"vm_id\":$vm_id}"
cleanup_job="${jid:-orphan-$vm_id}"
cleanup_attempt="${att:-0}"
cleanup_rec="$(jq -n --arg j "$cleanup_job" --arg a "$cleanup_attempt" \
--arg v "$vm_id" --arg vn "$(printf '%s' "$vm" | jq -r '.name // ""')" \
'{job_id:$j, run_attempt:$a, repo:"", label:"", created_at:"0",
ttl_min:0, vm_id:($v|tonumber), vm_name:$vn,
bootstrapped:false, status:"cleanup_pending"}')"
if gcr_vm_cleanup_start "$cleanup_job" "$cleanup_attempt" \
"$cleanup_rec" orphan false; then
gcr_event "orphan-vm-destroyed" "$cleanup_job" "{\"vm_id\":$vm_id}"
else
gcr_event "vm-cleanup-pending" "$cleanup_job" \
"{\"vm_id\":$vm_id,\"reason\":\"orphan\"}"
fi
fi
i=$((i + 1))
done
@@ -149,7 +173,7 @@ gcr_alloc_deferred() {
return 0
}
set -- $profile
server_type="$1"; ttl_min="$2"; rate="$3"
server_type="$1"; ttl_min="$2"
gcr_lock_acquire admission || {
gcr_lock_release "$key"
@@ -190,11 +214,6 @@ gcr_alloc_deferred() {
return 0
fi
gcr_budget_can_add "$rate" "$ttl_min" || {
gcr_lock_release admission
gcr_lock_release "$key"
return 0
}
reg_token="$(gcr_gitea_registration_token "$repo")" || {
gcr_lock_release admission
gcr_lock_release "$key"
@@ -203,23 +222,19 @@ gcr_alloc_deferred() {
vm_name="gcr-${job_id}-${attempt}"
created_at="$(gcr_now_epoch)"
vm_id="$(gcr_vm_create "$vm_name" "$label" "$server_type" "$ttl_min" \
"$reg_token" "$job_id" "$attempt" "$repo")" && [ -n "$vm_id" ] || {
create_status=0
created="$(gcr_vm_create "$vm_name" "$label" "$server_type" "$ttl_min" \
"$reg_token" "$job_id" "$attempt" "$repo")" || create_status="$?"
if [ "$create_status" -ne 0 ] || [ -z "$created" ]; then
gcr_lock_release admission
gcr_lock_release "$key"
return 0
}
gcr_budget_add "$rate" "$ttl_min"
rec="$(jq -n --arg j "$job_id" --arg a "$attempt" --arg r "$repo" \
--arg l "$label" --arg t "$created_at" --arg v "$vm_id" \
--arg vn "$vm_name" --arg ttl "$ttl_min" \
'{job_id:$j, run_attempt:$a, repo:$r, label:$l,
created_at:$t, ttl_min:($ttl|tonumber), vm_id:($v|tonumber),
vm_name:$vn, bootstrapped:false, status:"pending_vm"}')"
if ! gcr_record_put "$job_id" "$attempt" "$rec"; then
gcr_vm_destroy "$vm_id" || true
fi
set -- $created
vm_id="$1"; actual_server_type="$2"; actual_rate="$3"
if ! gcr_vm_record_created "$job_id" "$attempt" "$repo" "$label" \
"$created_at" "$vm_id" "$vm_name" "$ttl_min" \
"$actual_server_type" "$actual_rate"; then
gcr_lock_release admission
gcr_lock_release "$key"
return 0
@@ -230,6 +245,83 @@ gcr_alloc_deferred() {
gcr_log info --ns=alloc "deferred job=$job_id allocated vm=$vm_id"
}
gcr_sweep_cleanup_pending() {
for f in $(gcr_active_records); do
rec="$(cat "$f")"
[ "$(gcr_record_field "$rec" status)" = "cleanup_pending" ] || continue
job_id="$(gcr_record_field "$rec" job_id)"
attempt="$(gcr_record_field "$rec" run_attempt)"
key="$(gcr_alloc_key "$job_id" "$attempt")"
gcr_lock_acquire "$key" || continue
if ! gcr_lock_acquire admission; then
gcr_lock_release "$key"
continue
fi
rec="$(gcr_record_get "$job_id" "$attempt")"
if [ "$(gcr_record_field "$rec" status)" = "cleanup_pending" ]; then
gcr_vm_cleanup_pending "$job_id" "$attempt" "$rec" || true
fi
gcr_lock_release admission
gcr_lock_release "$key"
done
}
gcr_sweep_create_ambiguous() {
for f in $(gcr_active_records); do
rec="$(cat "$f")"
[ "$(gcr_record_field "$rec" status)" = "create_ambiguous" ] || continue
job_id="$(gcr_record_field "$rec" job_id)"
attempt="$(gcr_record_field "$rec" run_attempt)"
key="$(gcr_alloc_key "$job_id" "$attempt")"
gcr_lock_acquire "$key" || continue
if ! gcr_lock_acquire admission; then
gcr_lock_release "$key"
continue
fi
rec="$(gcr_record_get "$job_id" "$attempt")"
if [ "$(gcr_record_field "$rec" status)" != "create_ambiguous" ]; then
gcr_lock_release admission
gcr_lock_release "$key"
continue
fi
find_status=0
found_vm_id="$(gcr_vm_find_created \
"$(gcr_record_field "$rec" vm_name)" "$job_id" "$attempt" \
"$(gcr_record_field "$rec" label)" \
"$(gcr_record_field "$rec" server_type)" \
"$(gcr_record_field "$rec" candidate_location)" \
"$(gcr_record_field "$rec" candidate_arch)")" || find_status="$?"
case "$find_status" in
0)
rec="$(printf '%s' "$rec" | jq -c --arg vm "$found_vm_id" \
'.vm_id = ($vm | tonumber)
| .status = "pending_vm"
| .bootstrapped = false
| del(.create_http, .create_curl_status,
.candidate_location, .candidate_arch)')"
if gcr_record_put "$job_id" "$attempt" "$rec"; then
gcr_event "vm-create-recovered" "$job_id" \
"{\"vm_id\":$found_vm_id,\"label\":\"$(gcr_record_field "$rec" label)\"}"
fi
;;
1)
rec="$(printf '%s' "$rec" | jq -c \
'.status = "cleanup_pending"
| .cleanup_reason = "ambiguous-create-absent"
| .cleanup_refund_budget = true
| .cleanup_vm_destroyed = true
| .cleanup_budget_released = false')"
if gcr_record_put "$job_id" "$attempt" "$rec"; then
gcr_vm_cleanup_pending "$job_id" "$attempt" "$rec" || true
fi
;;
2) ;;
esac
gcr_lock_release admission
gcr_lock_release "$key"
done
}
gcr_retry_deferred() {
for f in $(gcr_active_records); do
rec="$(cat "$f")"
@@ -343,10 +435,17 @@ gcr_bootstrap_pending() {
gcr_vm_collect_diagnostics "$vm_id" "$ip" "$job_id" "$state" || true
;;
esac
gcr_vm_destroy "$vm_id" || true
gcr_event "vm-destroyed" "$job_id" "{\"vm_id\":$vm_id,\"reason\":\"pending-job-completed\",\"state\":\"$state\"}"
if gcr_vm_cleanup_start "$job_id" "$attempt" "$rec" \
pending-job-completed false; then
gcr_event "vm-destroyed" "$job_id" \
"{\"vm_id\":$vm_id,\"reason\":\"pending-job-completed\",\"state\":\"$state\"}"
else
gcr_event "vm-cleanup-pending" "$job_id" \
"{\"vm_id\":$vm_id,\"reason\":\"pending-job-completed\"}"
fi
else
gcr_record_del "$job_id" "$attempt"
fi
gcr_record_del "$job_id" "$attempt"
gcr_lock_release "$key"
continue
;;
@@ -418,10 +517,15 @@ gcr_reap_finished_jobs() {
if ! gcr_gitea_runner_disabled "$repo" "$runner_name" true \
|| ! gcr_vm_runner_service "$vm_id" stop; then
gcr_lock_release idle-pool
gcr_vm_destroy "$vm_id" || true
gcr_record_del "$job_id" "$attempt"
if gcr_vm_cleanup_start "$job_id" "$attempt" "$rec" \
idle-stop-failed false; then
gcr_event "vm-destroyed" "$job_id" \
"{\"vm_id\":$vm_id,\"reason\":\"idle-stop-failed\",\"via\":\"reconcile\"}"
else
gcr_event "vm-cleanup-pending" "$job_id" \
"{\"vm_id\":$vm_id,\"reason\":\"idle-stop-failed\",\"via\":\"reconcile\"}"
fi
gcr_lock_release "$key"
gcr_event "vm-destroyed" "$job_id" "{\"vm_id\":$vm_id,\"reason\":\"idle-stop-failed\",\"via\":\"reconcile\"}"
continue
fi
gcr_record_put "$job_id" "$attempt" "$idle_rec"
@@ -442,10 +546,17 @@ gcr_reap_finished_jobs() {
gcr_vm_collect_diagnostics "$vm_id" "$ip" "$job_id" "$state" || true
;;
esac
gcr_vm_destroy "$vm_id" || true
gcr_event "vm-destroyed" "$job_id" "{\"vm_id\":$vm_id,\"reason\":\"job-completed\",\"state\":\"$state\"}"
if gcr_vm_cleanup_start "$job_id" "$attempt" "$rec" \
job-completed false; then
gcr_event "vm-destroyed" "$job_id" \
"{\"vm_id\":$vm_id,\"reason\":\"job-completed\",\"state\":\"$state\"}"
else
gcr_event "vm-cleanup-pending" "$job_id" \
"{\"vm_id\":$vm_id,\"reason\":\"job-completed\",\"state\":\"$state\"}"
fi
else
gcr_record_del "$job_id" "$attempt"
fi
gcr_record_del "$job_id" "$attempt"
gcr_lock_release "$key"
;;
esac
@@ -453,6 +564,8 @@ gcr_reap_finished_jobs() {
}
gcr_tick() {
gcr_sweep_create_ambiguous
gcr_sweep_cleanup_pending
gcr_sweep_ttl
gcr_reap_finished_jobs
gcr_sweep_orphan_vms
package/gitea-runner-controller/decide.sh
+18 -4
View File
@@ -41,7 +41,14 @@ gcr_label_ttl() {
gcr_label_candidates() {
label="$1"
case "$label" in
ubuntu-latest|gross-x86)
ubuntu-latest)
printf '%s\n' \
'cx23 nbg1 amd64' 'cx23 fsn1 amd64' 'cx23 hel1 amd64' \
'cx33 nbg1 amd64' 'cx33 fsn1 amd64' 'cx33 hel1 amd64' \
'cx43 nbg1 amd64' 'cx43 fsn1 amd64' 'cx43 hel1 amd64' \
'cx53 nbg1 amd64' 'cx53 fsn1 amd64' 'cx53 hel1 amd64'
;;
gross-x86)
printf '%s\n' \
'cx53 nbg1 amd64' 'cx53 fsn1 amd64' 'cx53 hel1 amd64' \
'cx43 nbg1 amd64' 'cx43 fsn1 amd64' 'cx43 hel1 amd64' \
@@ -65,7 +72,14 @@ gcr_label_candidates() {
'cax41 nbg1 arm64' 'cax41 fsn1 arm64' 'cax41 hel1 arm64' \
'cx43 nbg1 amd64' 'cx43 fsn1 amd64' 'cx43 hel1 amd64'
;;
nix|gross-nix-x86)
nix)
printf '%s\n' \
'cx23 nbg1 amd64' 'cx23 fsn1 amd64' 'cx23 hel1 amd64' \
'cx33 nbg1 amd64' 'cx33 fsn1 amd64' 'cx33 hel1 amd64' \
'cx43 nbg1 amd64' 'cx43 fsn1 amd64' 'cx43 hel1 amd64' \
'cx53 nbg1 amd64' 'cx53 fsn1 amd64' 'cx53 hel1 amd64'
;;
gross-nix-x86)
printf '%s\n' \
'cx53 nbg1 amd64' 'cx53 fsn1 amd64' 'cx53 hel1 amd64' \
'cx43 nbg1 amd64' 'cx43 fsn1 amd64' 'cx43 hel1 amd64' \
@@ -148,7 +162,7 @@ gcr_count_active() {
for f in $(gcr_active_records); do
status="$(gcr_record_field "$(cat "$f")" status)"
case "$status" in
pending_vm|vm_active) count=$((count + 1)) ;;
pending_vm|vm_active|cleanup_pending|create_ambiguous) count=$((count + 1)) ;;
esac
done
printf '%s' "$count"
@@ -160,7 +174,7 @@ gcr_count_active_repo() {
for f in $(gcr_active_records); do
rec="$(cat "$f")"
case "$(gcr_record_field "$rec" status)" in
pending_vm|vm_active) ;;
pending_vm|vm_active|cleanup_pending|create_ambiguous) ;;
*) continue ;;
esac
[ "$(gcr_record_field "$rec" repo)" = "$repo" ] && count=$((count + 1))
package/gitea-runner-controller/hcloud.sh
+179 -6
View File
@@ -36,6 +36,7 @@ gcr_hcloud_req() {
method="$1"; path="$2"; body="${3:-}"
token="$(gcr_hcloud_token)" || return 1
GCR_LAST_BODY="$(mktemp "${TMPDIR:-/tmp}/gcr-resp.XXXXXX")"
curl_status=0
if [ -n "$body" ]; then
code="$(printf '%s' "$body" | curl -sS -X "$method" \
-H "Authorization: Bearer $token" \
@@ -43,13 +44,20 @@ gcr_hcloud_req() {
--data-binary @- \
-o "$GCR_LAST_BODY" \
-w '%{http_code}' \
"$GCR_API$path")"
"$GCR_API$path")" || curl_status="$?"
else
code="$(curl -sS -X "$method" \
-H "Authorization: Bearer $token" \
-o "$GCR_LAST_BODY" \
-w '%{http_code}' \
"$GCR_API$path")"
"$GCR_API$path")" || curl_status="$?"
fi
case "$code" in ''|*[!0-9]*) code=000 ;; esac
GCR_LAST_CURL_STATUS="$curl_status"
GCR_LAST_HTTP="$code"
if [ "$curl_status" -ne 0 ]; then
gcr_log warn --ns=hcloud "transport failed path=$path curl=$curl_status http=$code"
return 1
fi
case "$code" in 2??) return 0 ;; esac
gcr_log warn --ns=hcloud "request failed path=$path http=$code body=$(head -c 200 "$GCR_LAST_BODY" | gcr_redact)"
@@ -62,6 +70,64 @@ gcr_vm_list_managed() {
fi
}
# Exact deterministic create identity. Exit 0 = one match (prints id),
# 1 = confirmed absent, 2 = lookup failed or identity invariant violated.
gcr_vm_find_created() {
find_name="$1"; find_job="$2"; find_attempt="$3"; find_label="$4"
find_type="$5"; find_location="$6"; find_arch="$7"
if ! gcr_hcloud_req GET "/servers?name=$find_name"; then
return 2
fi
find_matches="$(jq -c \
--arg name "$find_name" --arg job "$find_job" --arg attempt "$find_attempt" \
--arg label "$find_label" --arg type "$find_type" \
--arg location "$find_location" --arg arch "$find_arch" \
'[.servers[] | select(
.name == $name
and .labels["gitea-runner-controller"] == "managed"
and .labels["gcr.job-id"] == $job
and .labels["gcr.run-attempt"] == $attempt
and .labels["gcr.label"] == $label
and .labels["gcr.location"] == $location
and .labels["gcr.arch"] == $arch
and ((.server_type.name // .server_type) == $type))]' \
"$GCR_LAST_BODY")" || return 2
find_count="$(printf '%s' "$find_matches" | jq 'length')" || return 2
case "$find_count" in
0) return 1 ;;
1) printf '%s' "$find_matches" | jq -r '.[0].id' ;;
*) return 2 ;;
esac
}
gcr_create_explicitly_rejected() {
case "$1" in
400|401|403|404|405|409|412|422|423) return 0 ;;
*) return 1 ;;
esac
}
gcr_vm_record_create_ambiguous() {
ambiguous_name="$1"; ambiguous_label="$2"; ambiguous_ttl="$3"
ambiguous_job="$4"; ambiguous_attempt="$5"; ambiguous_repo="$6"
ambiguous_type="$7"; ambiguous_rate="$8"; ambiguous_location="$9"
shift 9; ambiguous_arch="$1"; ambiguous_http="$2"; ambiguous_curl="$3"
ambiguous_rec="$(jq -n --arg j "$ambiguous_job" --arg a "$ambiguous_attempt" \
--arg r "$ambiguous_repo" --arg l "$ambiguous_label" \
--arg t "$(gcr_now_epoch)" --arg vn "$ambiguous_name" \
--arg ttl "$ambiguous_ttl" --arg st "$ambiguous_type" \
--arg rate "$ambiguous_rate" --arg loc "$ambiguous_location" \
--arg arch "$ambiguous_arch" --arg http "$ambiguous_http" \
--arg curl "$ambiguous_curl" \
'{job_id:$j, run_attempt:$a, repo:$r, label:$l,
created_at:$t, ttl_min:($ttl|tonumber), vm_id:"", vm_name:$vn,
server_type:$st, budget_rate:$rate, candidate_location:$loc,
candidate_arch:$arch, create_http:$http, create_curl_status:$curl,
bootstrapped:false,
status:"create_ambiguous"}')"
gcr_record_put "$ambiguous_job" "$ambiguous_attempt" "$ambiguous_rec"
}
gcr_vm_build_userdata() {
vm_name="$1"; label="$2"; reg_token="$3"
@@ -163,9 +229,10 @@ runcmd:
}
# gcr_vm_create NAME LABEL SERVER_TYPE TTL_MIN REG_TOKEN JOB_ID ATTEMPT REPO
# Prints new server id.
# Caller holds admission lock. Reserves each affordable candidate before its
# create request; prints new server id, actual server type, and reserved rate.
gcr_vm_create() {
vm_name="$1"; label="$2"; server_type="$3"; ttl_min="$4"
vm_name="$1"; label="$2"; ttl_min="$4"
reg_token="$5"; job_id="$6"; attempt="$7"; repo="$8"
ttl_min="$(gcr_label_ttl "$label")" || return 1
@@ -174,8 +241,18 @@ gcr_vm_create() {
while read -r candidate_type candidate_loc candidate_arch; do
[ -n "${candidate_type:-}" ] || continue
candidate_n=$((candidate_n + 1))
candidate_rate="$(gcr_server_hourly_rate "$candidate_type")" || continue
if ! gcr_budget_can_add "$candidate_rate" "$ttl_min"; then
gcr_log info --ns=hcloud "skip candidate[$candidate_n] label=$label type=$candidate_type over budget"
continue
fi
if ! gcr_budget_add "$candidate_rate" "$ttl_min"; then
gcr_log error --ns=hcloud "budget reservation write failed label=$label type=$candidate_type"
return 1
fi
image_id="$(gcr_image_id_for_arch "$candidate_arch" "$label")" || {
gcr_log warn --ns=hcloud "skip candidate[$candidate_n] label=$label arch=$candidate_arch no image"
gcr_budget_sub "$candidate_rate" "$ttl_min" || return 1
continue
}
payload="$(jq -n \
@@ -203,9 +280,34 @@ gcr_vm_create() {
"gcr.created-at":$ts, "gcr.ttl-min":$ttl}}')"
gcr_log info --ns=hcloud "try candidate[$candidate_n] label=$label type=$candidate_type arch=$candidate_arch loc=$candidate_loc"
if gcr_hcloud_req POST /servers "$payload"; then
jq -r '.server.id' "$GCR_LAST_BODY"
printf '%s %s %s\n' \
"$(jq -r '.server.id' "$GCR_LAST_BODY")" "$candidate_type" "$candidate_rate"
return 0
fi
create_http="${GCR_LAST_HTTP:-000}"
create_curl="${GCR_LAST_CURL_STATUS:-0}"
find_status=0
found_vm_id="$(gcr_vm_find_created "$vm_name" "$job_id" "$attempt" \
"$label" "$candidate_type" "$candidate_loc" "$candidate_arch")" \
|| find_status="$?"
if [ "$find_status" -eq 0 ]; then
printf '%s %s %s\n' "$found_vm_id" "$candidate_type" "$candidate_rate"
return 0
fi
if [ "$find_status" -eq 1 ] \
&& gcr_create_explicitly_rejected "$create_http"; then
if ! gcr_budget_sub "$candidate_rate" "$ttl_min"; then
gcr_log error --ns=hcloud "budget reservation rollback failed label=$label type=$candidate_type"
return 1
fi
else
if ! gcr_vm_record_create_ambiguous "$vm_name" "$label" "$ttl_min" \
"$job_id" "$attempt" "$repo" "$candidate_type" "$candidate_rate" \
"$candidate_loc" "$candidate_arch" "$create_http" "$create_curl"; then
gcr_log error --ns=hcloud "cannot persist ambiguous create job=$job_id type=$candidate_type"
fi
return 2
fi
if [ "$candidate_n" -le 3 ]; then
sleep 5
else
@@ -217,7 +319,7 @@ EOF
return 1
}
# gcr_vm_destroy SERVER_ID — idempotent best-effort destroy.
# gcr_vm_destroy SERVER_ID — success means DELETE returned HTTP 2xx.
gcr_vm_destroy() {
if ! gcr_hcloud_req DELETE "/servers/$1"; then
gcr_log warn --ns=hcloud "destroy failed or already gone: server $1"
@@ -225,6 +327,77 @@ gcr_vm_destroy() {
fi
}
# Only cleanup records establish prior ownership, making DELETE 404 a
# confirmed-absent success rather than an ambiguous lookup failure.
gcr_vm_destroy_owned() {
gcr_vm_destroy "$1" && return 0
[ "${GCR_LAST_HTTP:-}" = "404" ]
}
# Caller holds the lifecycle path's existing ownership locks.
gcr_vm_cleanup_pending() {
cleanup_job="$1"; cleanup_attempt="$2"; cleanup_rec="$3"
cleanup_vm_id="$(gcr_record_field "$cleanup_rec" vm_id)"
if [ "$(gcr_record_field "$cleanup_rec" cleanup_vm_destroyed)" != "true" ]; then
gcr_vm_destroy_owned "$cleanup_vm_id" || return 1
cleanup_rec="$(printf '%s' "$cleanup_rec" | jq -c '.cleanup_vm_destroyed = true')"
gcr_record_put "$cleanup_job" "$cleanup_attempt" "$cleanup_rec" || return 1
fi
if [ "$(gcr_record_field "$cleanup_rec" cleanup_refund_budget)" = "true" ] \
&& [ "$(gcr_record_field "$cleanup_rec" cleanup_budget_released)" != "true" ]; then
cleanup_rate="$(gcr_record_field "$cleanup_rec" budget_rate)"
cleanup_ttl="$(gcr_record_field "$cleanup_rec" ttl_min)"
cleanup_refund_key="$(gcr_alloc_key "$cleanup_job" "$cleanup_attempt")"
gcr_budget_refund_once "$cleanup_refund_key" "$cleanup_rate" "$cleanup_ttl" \
|| return 1
cleanup_rec="$(printf '%s' "$cleanup_rec" | jq -c '.cleanup_budget_released = true')"
gcr_record_put "$cleanup_job" "$cleanup_attempt" "$cleanup_rec" || return 1
fi
gcr_record_del "$cleanup_job" "$cleanup_attempt"
}
# Persist intent before DELETE. Normal lifecycle teardown never changes budget;
# failed creation passes REFUND_BUDGET=true to release its unused reservation.
gcr_vm_cleanup_start() {
cleanup_job="$1"; cleanup_attempt="$2"; cleanup_source="$3"
cleanup_reason="$4"; cleanup_refund="$5"
cleanup_rec="$(printf '%s' "$cleanup_source" | jq -c \
--arg reason "$cleanup_reason" --argjson refund "$cleanup_refund" \
'.status = "cleanup_pending"
| .cleanup_reason = $reason
| .cleanup_refund_budget = $refund
| .cleanup_vm_destroyed = false
| .cleanup_budget_released = false')"
gcr_record_put "$cleanup_job" "$cleanup_attempt" "$cleanup_rec" || return 1
gcr_vm_cleanup_pending "$cleanup_job" "$cleanup_attempt" "$cleanup_rec"
}
# Caller holds allocation and admission locks. A failed primary write first
# persists cleanup ownership; reservation is released only after destroy.
gcr_vm_record_created() {
record_job="$1"; record_attempt="$2"; record_repo="$3"; record_label="$4"
record_created="$5"; record_vm_id="$6"; record_vm_name="$7"
record_ttl="$8"; record_type="$9"; shift 9; record_rate="$1"
record_rec="$(jq -n --arg j "$record_job" --arg a "$record_attempt" \
--arg r "$record_repo" --arg l "$record_label" --arg t "$record_created" \
--arg v "$record_vm_id" --arg vn "$record_vm_name" --arg ttl "$record_ttl" \
--arg st "$record_type" --arg rate "$record_rate" \
'{job_id:$j, run_attempt:$a, repo:$r, label:$l,
created_at:$t, ttl_min:($ttl|tonumber), vm_id:($v|tonumber),
vm_name:$vn, server_type:$st, budget_rate:$rate,
bootstrapped:false, status:"pending_vm"}')"
gcr_record_put "$record_job" "$record_attempt" "$record_rec" && return 0
if ! gcr_vm_cleanup_start "$record_job" "$record_attempt" "$record_rec" \
state-write-failed true; then
cleanup_rec="$(gcr_record_get "$record_job" "$record_attempt")"
[ "$(gcr_record_field "$cleanup_rec" status)" = "cleanup_pending" ] && return 1
gcr_log error --ns=alloc "cannot persist cleanup record job=$record_job vm=$record_vm_id"
return 1
fi
return 1
}
gcr_vm_public_ip() {
# gcr_vm_public_ip SERVER_ID -> ipv4 or empty
if gcr_hcloud_req GET "/servers/$1"; then
package/gitea-runner-controller/state.sh
+54 -3
View File
@@ -257,6 +257,21 @@ gcr_budget_can_add() {
return 0
}
# Atomic replacement preserves last valid total when a write fails.
gcr_budget_write() {
gcr_budget_file="$1"; gcr_budget_value="$2"
gcr_budget_tmp="$(mktemp "$(dirname "$gcr_budget_file")/.budget.XXXXXX")" \
|| return 1
if ! printf '%s\n' "$gcr_budget_value" > "$gcr_budget_tmp"; then
rm -f "$gcr_budget_tmp"
return 1
fi
if ! mv -f "$gcr_budget_tmp" "$gcr_budget_file"; then
rm -f "$gcr_budget_tmp"
return 1
fi
}
# Caller holds admission lock and has already checked gcr_budget_can_add.
gcr_budget_add() {
rate="$1"; ttl_min="$2"
@@ -264,10 +279,46 @@ gcr_budget_add() {
file="$GCR_STATE_DIR/budget/$month"
current="$(cat "$file" 2>/dev/null || echo 0)"
projected="$(awk -v c="$current" -v r="$rate" -v t="$ttl_min" 'BEGIN {printf "%.4f", c + r * t / 60}')"
printf '%s\n' "$projected" > "$file"
return 0
gcr_budget_write "$file" "$projected"
}
# Caller holds admission lock and is rolling back a matching budget addition.
gcr_budget_sub() {
rate="$1"; ttl_min="$2"
month="$(date -u '+%Y-%m')"
file="$GCR_STATE_DIR/budget/$month"
current="$(cat "$file" 2>/dev/null || echo 0)"
projected="$(awk -v c="$current" -v r="$rate" -v t="$ttl_min" \
'BEGIN {v = c - r * t / 60; if (v < 0) v = 0; printf "%.4f", v}')"
gcr_budget_write "$file" "$projected"
}
# Claim is durable before credit. Existing claim means credit is consumed:
# it may have completed, or it may have leaked fail-closed after a crash.
# Never subtract twice when outcome between aggregate and state writes is unknown.
gcr_budget_refund_once() {
refund_key="$1"; refund_rate="$2"; refund_ttl="$3"
refund_month="$(date -u '+%Y-%m')"
refund_root="$GCR_STATE_DIR/budget/refunds/$refund_month"
refund_claim="$refund_root/$refund_key"
mkdir -p "$refund_root" || return 1
if ! mkdir "$refund_claim" 2>/dev/null; then
if [ -d "$refund_claim" ]; then
if [ ! -f "$refund_claim/status" ]; then
gcr_log error --ns=budget \
"refund outcome uncertain key=$refund_key; retaining fail-closed claim"
fi
return 0
fi
return 1
fi
if ! printf '%s %s\n' "$refund_rate" "$refund_ttl" > "$refund_claim/intent"; then
return 1
fi
gcr_budget_sub "$refund_rate" "$refund_ttl" || return 1
printf 'refunded\n' > "$refund_claim/status" || return 1
}
gcr_active_records() {
grep -El '"status"[[:space:]]*:[[:space:]]*"(pending_vm|vm_active|idle_vm|deferred)"' "$GCR_STATE_DIR"/jobs/*.json 2>/dev/null || true
grep -El '"status"[[:space:]]*:[[:space:]]*"(pending_vm|vm_active|idle_vm|deferred|cleanup_pending|create_ambiguous)"' "$GCR_STATE_DIR"/jobs/*.json 2>/dev/null || true
}
package/gitea-runner-controller/webhook.sh
+35 -34
View File
@@ -116,7 +116,7 @@ gcr_alloc() {
fi
set -- $profile
server_type="$1"; ttl_min="$2"; rate="$3"
server_type="$1"; ttl_min="$2"
if ! gcr_lock_acquire admission; then
rec="$(jq -n --arg j "$job_id" --arg a "$attempt" --arg r "$repo" \
@@ -183,15 +183,6 @@ gcr_alloc() {
return 0
fi
if ! gcr_budget_can_add "$rate" "$ttl_min"; then
gcr_record_del "$job_id" "$attempt"
gcr_lock_release admission
gcr_lock_release "$key"
gcr_event "budget-refused" "$job_id" "{\"rate\":$rate,\"ttl_min\":$ttl_min}"
RESPONSE_CODE=202; RESPONSE_BODY="refused: monthly budget exhausted"
return 0
fi
reg_token="$(gcr_gitea_registration_token "$repo")" || {
gcr_record_del "$job_id" "$attempt"
gcr_lock_release admission
@@ -203,26 +194,27 @@ gcr_alloc() {
vm_name="gcr-${job_id}-${attempt}"
created_at="$(gcr_now_epoch)"
vm_id="$(gcr_vm_create "$vm_name" "$label" "$server_type" "$ttl_min" \
"$reg_token" "$job_id" "$attempt" "$repo")" || {
gcr_record_del "$job_id" "$attempt"
create_status=0
created="$(gcr_vm_create "$vm_name" "$label" "$server_type" "$ttl_min" \
"$reg_token" "$job_id" "$attempt" "$repo")" || create_status="$?"
if [ "$create_status" -ne 0 ]; then
if [ "$create_status" -eq 2 ]; then
gcr_event "vm-create-ambiguous" "$job_id" "{}"
RESPONSE_CODE=202; RESPONSE_BODY="VM creation pending recovery"
else
gcr_record_del "$job_id" "$attempt"
gcr_event "vm-create-failed" "$job_id" "{}"
RESPONSE_CODE=202; RESPONSE_BODY="VM creation failed"
fi
gcr_lock_release admission
gcr_lock_release "$key"
gcr_event "vm-create-failed" "$job_id" "{}"
RESPONSE_CODE=202; RESPONSE_BODY="VM creation failed"
return 0
}
gcr_budget_add "$rate" "$ttl_min"
rec="$(jq -n --arg j "$job_id" --arg a "$attempt" --arg r "$repo" \
--arg l "$label" --arg t "$created_at" --arg v "$vm_id" \
--arg vn "$vm_name" --arg ttl "$ttl_min" \
'{job_id:$j, run_attempt:$a, repo:$r, label:$l,
created_at:$t, ttl_min:($ttl|tonumber), vm_id:($v|tonumber),
vm_name:$vn, bootstrapped:false, status:"pending_vm"}')"
if ! gcr_record_put "$job_id" "$attempt" "$rec"; then
gcr_vm_destroy "$vm_id" || true
fi
set -- $created
vm_id="$1"; actual_server_type="$2"; actual_rate="$3"
if ! gcr_vm_record_created "$job_id" "$attempt" "$repo" "$label" \
"$created_at" "$vm_id" "$vm_name" "$ttl_min" \
"$actual_server_type" "$actual_rate"; then
gcr_lock_release admission
gcr_lock_release "$key"
RESPONSE_CODE=202; RESPONSE_BODY="VM state write failed"
@@ -266,10 +258,14 @@ gcr_deallocate() {
if ! gcr_gitea_runner_disabled "$(gcr_record_field "$rec" repo)" "$runner_name" true \
|| ! gcr_vm_runner_service "$vm_id" stop; then
gcr_lock_release idle-pool
gcr_vm_destroy "$vm_id" || true
gcr_record_del "$job_id" "$attempt"
if gcr_vm_cleanup_start "$job_id" "$attempt" "$rec" idle-stop-failed false; then
gcr_event "vm-destroyed" "$job_id" \
"{\"vm_id\":$vm_id,\"reason\":\"idle-stop-failed\"}"
else
gcr_event "vm-cleanup-pending" "$job_id" \
"{\"vm_id\":$vm_id,\"reason\":\"idle-stop-failed\"}"
fi
gcr_lock_release "$key"
gcr_event "vm-destroyed" "$job_id" "{\"vm_id\":$vm_id,\"reason\":\"idle-stop-failed\"}"
return 0
fi
gcr_record_put "$job_id" "$attempt" "$idle_rec"
@@ -288,11 +284,16 @@ gcr_deallocate() {
gcr_vm_collect_diagnostics "$vm_id" "$ip" "$job_id" "$new_status" || true
;;
esac
gcr_vm_destroy "$vm_id" || true
gcr_event "vm-destroyed" "$job_id" "{\"vm_id\":$vm_id,\"reason\":\"$new_status\"}"
if gcr_vm_cleanup_start "$job_id" "$attempt" "$rec" "$new_status" false; then
gcr_event "vm-destroyed" "$job_id" \
"{\"vm_id\":$vm_id,\"reason\":\"$new_status\"}"
else
gcr_event "vm-cleanup-pending" "$job_id" \
"{\"vm_id\":$vm_id,\"reason\":\"$new_status\"}"
fi
else
gcr_record_del "$job_id" "$attempt"
fi
gcr_record_del "$job_id" "$attempt"
gcr_lock_release "$key"
}