diff --git a/package/gitea-runner-controller/webhook.sh b/package/gitea-runner-controller/webhook.sh index 526fcd8c..358dfb72 100644 --- a/package/gitea-runner-controller/webhook.sh +++ b/package/gitea-runner-controller/webhook.sh @@ -155,6 +155,7 @@ gcr_alloc() { vm_id="$(gcr_record_field "$reused" vm_id)" vm_name="$(gcr_record_field "$reused" vm_name)" if gcr_vm_runner_service "$vm_id" start \ + && gcr_vm_runner_service "$vm_id" health \ && gcr_gitea_runner_disabled "$repo" "$vm_name" false; then reused="$(gcr_record_get "$job_id" "$attempt")" reused="$(printf '%s' "$reused" | jq -c '.bootstrapped = true | del(.reused_vm)')" @@ -247,54 +248,14 @@ gcr_deallocate() { ;; esac - vm_id="$(gcr_record_field "$rec" vm_id)" - if [ "$new_status" = "completed:success" ] \ - && idle_rec="$(gcr_record_idle_json "$rec")"; then - if ! gcr_lock_acquire idle-pool; then - gcr_lock_release "$key" - return 0 - fi - runner_name="$(gcr_record_field "$rec" vm_name)" - if ! gcr_gitea_runner_disabled "$(gcr_record_field "$rec" repo)" "$runner_name" true \ - || ! gcr_vm_runner_service "$vm_id" stop; then - gcr_lock_release idle-pool - if gcr_vm_cleanup_start "$job_id" "$attempt" "$rec" idle-stop-failed false; then - gcr_event "vm-destroyed" "$job_id" \ - "{\"vm_id\":$vm_id,\"reason\":\"idle-stop-failed\"}" - else - gcr_event "vm-cleanup-pending" "$job_id" \ - "{\"vm_id\":$vm_id,\"reason\":\"idle-stop-failed\"}" - fi - gcr_lock_release "$key" - return 0 - fi - gcr_record_put "$job_id" "$attempt" "$idle_rec" - idle_expires="$(gcr_record_field "$idle_rec" idle_expires_at)" - gcr_lock_release idle-pool - gcr_lock_release "$key" - gcr_event "vm-idle" "$job_id" "{\"vm_id\":$vm_id,\"expires_at\":$idle_expires}" - return 0 - fi - - if [ -n "$vm_id" ] && [ "$vm_id" != "null" ] && [ "$vm_id" != "0" ]; then - case "$new_status" in - completed:success|completed:cancelled|completed:skipped) ;; - completed:*) - ip="$(gcr_vm_public_ip "$vm_id" || true)" - gcr_vm_collect_diagnostics "$vm_id" "$ip" "$job_id" "$new_status" || true - ;; - esac - if gcr_vm_cleanup_start "$job_id" "$attempt" "$rec" "$new_status" false; then - gcr_event "vm-destroyed" "$job_id" \ - "{\"vm_id\":$vm_id,\"reason\":\"$new_status\"}" - else - gcr_event "vm-cleanup-pending" "$job_id" \ - "{\"vm_id\":$vm_id,\"reason\":\"$new_status\"}" - fi - else - gcr_record_del "$job_id" "$attempt" - fi + finish_status=0 + gcr_vm_finish_terminal "$job_id" "$attempt" "$rec" "$new_status" webhook \ + || finish_status="$?" gcr_lock_release "$key" + case "$finish_status" in + 0|2) return 0 ;; + *) return "$finish_status" ;; + esac } gcr_mark_in_progress() { diff --git a/test/package/gitea-runner-controller/test/webhook-diagnostics.sh b/test/package/gitea-runner-controller/test/webhook-diagnostics.sh index dd96f49e..1afad99e 100644 --- a/test/package/gitea-runner-controller/test/webhook-diagnostics.sh +++ b/test/package/gitea-runner-controller/test/webhook-diagnostics.sh @@ -28,7 +28,9 @@ gcr_vm_runner_service() { printf 'runner %s vm=%s\n' "$2" "$1" >> "$calls" } -gcr_gitea_runner_disabled() { :; } +gcr_gitea_runner_disabled() { + printf 'disabled runner=%s value=%s\n' "$2" "$3" >> "$calls" +} record_success='{"job_id":"201","run_attempt":"1","repo":"hinterland/hearth","label":"gross-nix-x86-perf","created_at":"0","ttl_min":480,"vm_id":51,"vm_name":"gcr-201-1","bootstrapped":true,"status":"vm_active"}' record_failure='{"job_id":"202","run_attempt":"1","repo":"hinterland/hearth","label":"gross-nix-x86-perf","created_at":"1","ttl_min":480,"vm_id":52,"vm_name":"gcr-202-1","bootstrapped":true,"status":"vm_active"}' @@ -38,8 +40,14 @@ gcr_record_put 202 1 "$record_failure" gcr_deallocate 201 1 completed:success gcr_deallocate 202 1 completed:failure -grep -q 'destroy vm=52' "$calls" grep -q 'diag vm=52 ip=192.0.2.52 job=202 reason=completed:failure' "$calls" +grep -q 'runner health vm=52' "$calls" +grep -q 'disabled runner=gcr-202-1 value=true' "$calls" +grep -q 'runner stop vm=52' "$calls" +test "$(grep -E '^(diag vm=52|runner health vm=52|disabled runner=gcr-202-1|runner stop vm=52)' "$calls")" = 'diag vm=52 ip=192.0.2.52 job=202 reason=completed:failure +runner health vm=52 +disabled runner=gcr-202-1 value=true +runner stop vm=52' if grep -q 'diag vm=51' "$calls"; then printf 'success webhook should not collect diagnostics\n' >&2 exit 1 @@ -49,12 +57,36 @@ if grep -q 'destroy vm=51' "$calls"; then printf 'successful webhook VM should remain idle until billing boundary\n' >&2 exit 1 fi +if grep -q 'destroy vm=52' "$calls"; then + printf 'failed bootstrapped webhook VM should remain idle until billing boundary\n' >&2 + exit 1 +fi jq -e 'select(.status == "idle_vm" and .idle_expires_at == 3600)' \ "$(gcr_record_path 201 1)" >/dev/null +jq -e 'select(.status == "idle_vm" and .idle_expires_at == 3601)' \ + "$(gcr_record_path 202 1)" >/dev/null +gcr_idle_record_usable "$(gcr_record_get 202 1)" idle_once="$(gcr_record_get 201 1)" +gcr_record_del 201 1 +gcr_lock_acquire "$(gcr_alloc_key 205 1)" +gcr_claim_idle 205 1 hinterland/hearth gross-nix-x86-perf +gcr_lock_release "$(gcr_alloc_key 205 1)" +jq -e 'select(.job_id == "205" and .vm_id == 52 and .status == "pending_vm" and .reused_vm == true)' \ + "$(gcr_record_path 205 1)" >/dev/null +gcr_record_del 205 1 +gcr_record_put 201 1 "$idle_once" gcr_deallocate 201 1 completed:success test "$(gcr_record_get 201 1)" = "$idle_once" -test ! -e "$(gcr_record_path 202 1)" + +# Cancelled terminal jobs use same retention policy without failure diagnostics. +record_cancelled='{"job_id":"207","run_attempt":"1","repo":"hinterland/hearth","label":"nix","created_at":"1","ttl_min":480,"vm_id":57,"vm_name":"gcr-207-1","bootstrapped":true,"status":"vm_active"}' +gcr_record_put 207 1 "$record_cancelled" +gcr_deallocate 207 1 completed:cancelled +test "$(gcr_record_field "$(gcr_record_get 207 1)" status)" = idle_vm +if grep -q 'diag vm=57' "$calls" || grep -q 'destroy vm=57' "$calls"; then + printf 'healthy cancelled VM must be retained without failure diagnostics\n' >&2 + exit 1 +fi calls_ip_fail="$GCR_STATE_DIR/calls-ip-fail" calls="$calls_ip_fail" @@ -64,20 +96,33 @@ gcr_record_put 203 1 "$record_ip_fail" gcr_vm_public_ip() { return 1 } +gcr_vm_runner_service() { + printf 'runner %s vm=%s\n' "$2" "$1" >> "$calls" + return 1 +} gcr_deallocate 203 1 completed:failure grep -q 'diag vm=53 ip= job=203 reason=completed:failure' "$calls_ip_fail" +grep -q 'runner health vm=53' "$calls_ip_fail" grep -q 'destroy vm=53' "$calls_ip_fail" test ! -e "$(gcr_record_path 203 1)" +if grep -q 'disabled runner=gcr-203-1' "$calls_ip_fail"; then + printf 'unhealthy terminal runner must not enter idle shutdown path\n' >&2 + exit 1 +fi -# Terminal webhook retains ownership and capacity until DELETE succeeds. +# Runner teardown failure destroys and retains ownership until DELETE succeeds. record_delete_fail='{"job_id":"204","run_attempt":"1","repo":"hinterland/hearth","label":"nix","created_at":"1","ttl_min":480,"vm_id":54,"vm_name":"gcr-204-1","bootstrapped":true,"status":"vm_active"}' gcr_record_put 204 1 "$record_delete_fail" gcr_vm_destroy() { printf 'destroy-failed vm=%s\n' "$1" >> "$calls_ip_fail" return 1 } +gcr_vm_runner_service() { + printf 'runner %s vm=%s\n' "$2" "$1" >> "$calls" + [ "$2" != stop ] +} gcr_deallocate 204 1 completed:failure cleanup_rec="$(gcr_record_get 204 1)" test "$(gcr_record_field "$cleanup_rec" status)" = cleanup_pending @@ -94,6 +139,19 @@ test "$(gcr_count_active)" = 0 grep -q '^destroy-failed vm=54$' "$calls_ip_fail" grep -q '^destroy-retry vm=54$' "$calls_ip_fail" +# Unbootstrapped terminal VMs are never retained. +gcr_vm_runner_service() { printf 'unexpected-runner %s\n' "$1" >> "$calls_ip_fail"; } +gcr_vm_destroy() { printf 'destroy-unbootstrapped vm=%s\n' "$1" >> "$calls_ip_fail"; } +record_unbootstrapped='{"job_id":"206","run_attempt":"1","repo":"hinterland/hearth","label":"nix","created_at":"1","ttl_min":480,"vm_id":56,"vm_name":"gcr-206-1","bootstrapped":false,"status":"pending_vm"}' +gcr_record_put 206 1 "$record_unbootstrapped" +gcr_deallocate 206 1 completed:failure +grep -q '^destroy-unbootstrapped vm=56$' "$calls_ip_fail" +test ! -e "$(gcr_record_path 206 1)" +if grep -q '^unexpected-runner 56$' "$calls_ip_fail"; then + printf 'unbootstrapped VM must bypass idle teardown\n' >&2 + exit 1 +fi + # Gitea emits zero-based run_attempt values for initial workflow jobs. gcr_read_request() { gcr_hdr_event_type=workflow_job