diff --git a/package/gitea-runner-controller/controller.sh b/package/gitea-runner-controller/controller.sh index 22dfd83a..4d5391a3 100644 --- a/package/gitea-runner-controller/controller.sh +++ b/package/gitea-runner-controller/controller.sh @@ -194,6 +194,7 @@ gcr_alloc_deferred() { reused="$(gcr_record_get "$job_id" "$attempt")" vm_id="$(gcr_record_field "$reused" vm_id)" if gcr_vm_runner_service "$vm_id" start \ + && gcr_vm_runner_service "$vm_id" health \ && gcr_gitea_runner_disabled "$repo" "$(gcr_record_field "$reused" vm_name)" false; then reused="$(gcr_record_get "$job_id" "$attempt")" reused="$(printf '%s' "$reused" | jq -c '.bootstrapped = true | del(.reused_vm)')" @@ -410,6 +411,7 @@ gcr_bootstrap_pending() { if [ "$(gcr_record_field "$rec" reused_vm)" = "true" ]; then if gcr_vm_runner_service "$vm_id" start \ + && gcr_vm_runner_service "$vm_id" health \ && gcr_gitea_runner_disabled "$repo" "$runner_name" false; then rec="$(printf '%s' "$rec" | jq -c '.bootstrapped = true | del(.reused_vm)')" gcr_record_put "$job_id" "$attempt" "$rec" @@ -506,58 +508,14 @@ gcr_reap_finished_jobs() { pending_vm|vm_active) ;; *) gcr_lock_release "$key"; continue ;; esac - vm_id="$(gcr_record_field "$rec" vm_id)" - if [ "$state" = "completed:success" ] \ - && idle_rec="$(gcr_record_idle_json "$rec")"; then - if ! gcr_lock_acquire idle-pool; then - gcr_lock_release "$key" - continue - fi - runner_name="$(gcr_record_field "$rec" vm_name)" - if ! gcr_gitea_runner_disabled "$repo" "$runner_name" true \ - || ! gcr_vm_runner_service "$vm_id" stop; then - gcr_lock_release idle-pool - if gcr_vm_cleanup_start "$job_id" "$attempt" "$rec" \ - idle-stop-failed false; then - gcr_event "vm-destroyed" "$job_id" \ - "{\"vm_id\":$vm_id,\"reason\":\"idle-stop-failed\",\"via\":\"reconcile\"}" - else - gcr_event "vm-cleanup-pending" "$job_id" \ - "{\"vm_id\":$vm_id,\"reason\":\"idle-stop-failed\",\"via\":\"reconcile\"}" - fi - gcr_lock_release "$key" - continue - fi - gcr_record_put "$job_id" "$attempt" "$idle_rec" - idle_expires="$(gcr_record_field "$idle_rec" idle_expires_at)" - gcr_lock_release idle-pool - gcr_lock_release "$key" - gcr_event "vm-idle" "$job_id" "{\"vm_id\":$vm_id,\"expires_at\":$idle_expires,\"via\":\"reconcile\"}" - gcr_log info --ns=sweep "job=$job_id succeeded, retaining vm=$vm_id until $idle_expires" - continue - fi - - gcr_log info --ns=sweep "job=$job_id terminal ($state), destroying vm=$vm_id" - if [ -n "$vm_id" ] && [ "$vm_id" != "0" ] && [ "$vm_id" != "null" ]; then - case "$state" in - completed:success|completed:cancelled|completed:skipped) ;; - *) - ip="$(gcr_vm_public_ip "$vm_id" || true)" - gcr_vm_collect_diagnostics "$vm_id" "$ip" "$job_id" "$state" || true - ;; - esac - if gcr_vm_cleanup_start "$job_id" "$attempt" "$rec" \ - job-completed false; then - gcr_event "vm-destroyed" "$job_id" \ - "{\"vm_id\":$vm_id,\"reason\":\"job-completed\",\"state\":\"$state\"}" - else - gcr_event "vm-cleanup-pending" "$job_id" \ - "{\"vm_id\":$vm_id,\"reason\":\"job-completed\",\"state\":\"$state\"}" - fi - else - gcr_record_del "$job_id" "$attempt" - fi + finish_status=0 + gcr_vm_finish_terminal "$job_id" "$attempt" "$rec" "$state" reconcile \ + || finish_status="$?" gcr_lock_release "$key" + case "$finish_status" in + 0|2) ;; + *) return "$finish_status" ;; + esac ;; esac done diff --git a/test/package/gitea-runner-controller/test/reap-diagnostics.sh b/test/package/gitea-runner-controller/test/reap-diagnostics.sh index 2a01d81d..0829d3fa 100644 --- a/test/package/gitea-runner-controller/test/reap-diagnostics.sh +++ b/test/package/gitea-runner-controller/test/reap-diagnostics.sh @@ -16,6 +16,7 @@ gcr_gitea_job_state() { case "$2" in 101) printf 'completed:success' ;; 102) printf 'completed:failure' ;; + 105) printf 'completed:skipped' ;; *) return 1 ;; esac } @@ -36,7 +37,9 @@ gcr_vm_runner_service() { printf 'runner %s vm=%s\n' "$2" "$1" >> "$calls" } -gcr_gitea_runner_disabled() { :; } +gcr_gitea_runner_disabled() { + printf 'disabled runner=%s value=%s\n' "$2" "$3" >> "$calls" +} record_success='{"job_id":"101","run_attempt":"1","repo":"hinterland/hearth","label":"gross-nix-x86-perf","created_at":"0","ttl_min":480,"vm_id":41,"vm_name":"gcr-101-1","bootstrapped":true,"status":"vm_active"}' record_failure='{"job_id":"102","run_attempt":"1","repo":"hinterland/hearth","label":"gross-nix-x86-perf","created_at":"1","ttl_min":480,"vm_id":42,"vm_name":"gcr-102-1","bootstrapped":true,"status":"vm_active"}' @@ -45,8 +48,14 @@ gcr_record_put 101 1 "$record_success" gcr_record_put 102 1 "$record_failure" gcr_reap_finished_jobs -grep -q 'destroy vm=42' "$calls" grep -q 'diag vm=42 ip=192.0.2.42 job=102 reason=completed:failure' "$calls" +grep -q 'runner health vm=42' "$calls" +grep -q 'disabled runner=gcr-102-1 value=true' "$calls" +grep -q 'runner stop vm=42' "$calls" +test "$(grep -E '^(diag vm=42|runner health vm=42|disabled runner=gcr-102-1|runner stop vm=42)' "$calls")" = 'diag vm=42 ip=192.0.2.42 job=102 reason=completed:failure +runner health vm=42 +disabled runner=gcr-102-1 value=true +runner stop vm=42' if grep -q 'diag vm=41' "$calls"; then printf 'success job should not collect diagnostics\n' >&2 exit 1 @@ -56,12 +65,28 @@ if grep -q 'destroy vm=41' "$calls"; then printf 'successful job VM should remain idle until billing boundary\n' >&2 exit 1 fi +if grep -q 'destroy vm=42' "$calls"; then + printf 'failed bootstrapped reconciled VM should remain idle until billing boundary\n' >&2 + exit 1 +fi jq -e 'select(.status == "idle_vm" and .idle_expires_at == 3600)' \ "$(gcr_record_path 101 1)" >/dev/null +jq -e 'select(.status == "idle_vm" and .idle_expires_at == 3601)' \ + "$(gcr_record_path 102 1)" >/dev/null +gcr_idle_record_usable "$(gcr_record_get 102 1)" idle_once="$(gcr_record_get 101 1)" gcr_reap_finished_jobs test "$(gcr_record_get 101 1)" = "$idle_once" -test ! -e "$(gcr_record_path 102 1)" + +# Skipped terminal jobs follow same healthy retention policy without diagnostics. +record_skipped='{"job_id":"105","run_attempt":"1","repo":"hinterland/hearth","label":"nix","created_at":"1","ttl_min":480,"vm_id":45,"vm_name":"gcr-105-1","bootstrapped":true,"status":"vm_active"}' +gcr_record_put 105 1 "$record_skipped" +gcr_reap_finished_jobs +test "$(gcr_record_field "$(gcr_record_get 105 1)" status)" = idle_vm +if grep -q 'diag vm=45' "$calls" || grep -q 'destroy vm=45' "$calls"; then + printf 'healthy skipped VM must be retained without failure diagnostics\n' >&2 + exit 1 +fi # Reaper keeps cleanup ownership after DELETE failure and retries next sweep. record_delete_fail='{"job_id":"104","run_attempt":"1","repo":"hinterland/hearth","label":"nix","created_at":"1","ttl_min":480,"vm_id":44,"vm_name":"gcr-104-1","bootstrapped":true,"status":"vm_active"}' @@ -76,6 +101,10 @@ gcr_vm_destroy() { printf 'destroy-failed vm=%s\n' "$1" >> "$calls" return 1 } +gcr_vm_runner_service() { + printf 'runner %s vm=%s\n' "$2" "$1" >> "$calls" + [ "$1" != 44 ] || [ "$2" != stop ] +} gcr_reap_finished_jobs test "$(gcr_record_field "$(gcr_record_get 104 1)" status)" = cleanup_pending test "$(gcr_count_active)" = 1 @@ -101,6 +130,10 @@ gcr_gitea_job_state() { gcr_vm_public_ip() { return 1 } +gcr_vm_runner_service() { + printf 'runner %s vm=%s\n' "$2" "$1" >> "$calls" + return 1 +} gcr_vm_destroy() { printf 'destroy vm=%s\n' "$1" >> "$calls" @@ -109,5 +142,6 @@ gcr_vm_destroy() { gcr_reap_finished_jobs grep -q 'diag vm=43 ip= job=103 reason=completed:failure' "$calls_ip_fail" +grep -q 'runner health vm=43' "$calls_ip_fail" grep -q 'destroy vm=43' "$calls_ip_fail" test ! -e "$(gcr_record_path 103 1)"