hinterland/hearth
2
2
Fork 1
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix: reconcile healthy terminal runners
runner nix smoke / nix label and flake smoke (push) Failing after 28s
Details

Browse Source
This commit is contained in:
yukkop
2026-09-11 09:19:04 +00:00
parent 279df769db
commit cc8a7cf80e
2 changed files with 46 additions and 54 deletions
Download Patch File Download Diff File Expand all files Collapse all files
test/package/gitea-runner-controller/test/reap-diagnostics.sh
+37 -3
View File
@@ -16,6 +16,7 @@ gcr_gitea_job_state() {
case "$2" in
101) printf 'completed:success' ;;
102) printf 'completed:failure' ;;
105) printf 'completed:skipped' ;;
*) return 1 ;;
esac
}
@@ -36,7 +37,9 @@ gcr_vm_runner_service() {
printf 'runner %s vm=%s\n' "$2" "$1" >> "$calls"
}
gcr_gitea_runner_disabled() { :; }
gcr_gitea_runner_disabled() {
printf 'disabled runner=%s value=%s\n' "$2" "$3" >> "$calls"
}
record_success='{"job_id":"101","run_attempt":"1","repo":"hinterland/hearth","label":"gross-nix-x86-perf","created_at":"0","ttl_min":480,"vm_id":41,"vm_name":"gcr-101-1","bootstrapped":true,"status":"vm_active"}'
record_failure='{"job_id":"102","run_attempt":"1","repo":"hinterland/hearth","label":"gross-nix-x86-perf","created_at":"1","ttl_min":480,"vm_id":42,"vm_name":"gcr-102-1","bootstrapped":true,"status":"vm_active"}'
@@ -45,8 +48,14 @@ gcr_record_put 101 1 "$record_success"
gcr_record_put 102 1 "$record_failure"
gcr_reap_finished_jobs
grep -q 'destroy vm=42' "$calls"
grep -q 'diag vm=42 ip=192.0.2.42 job=102 reason=completed:failure' "$calls"
grep -q 'runner health vm=42' "$calls"
grep -q 'disabled runner=gcr-102-1 value=true' "$calls"
grep -q 'runner stop vm=42' "$calls"
test "$(grep -E '^(diag vm=42|runner health vm=42|disabled runner=gcr-102-1|runner stop vm=42)' "$calls")" = 'diag vm=42 ip=192.0.2.42 job=102 reason=completed:failure
runner health vm=42
disabled runner=gcr-102-1 value=true
runner stop vm=42'
if grep -q 'diag vm=41' "$calls"; then
printf 'success job should not collect diagnostics\n' >&2
exit 1
@@ -56,12 +65,28 @@ if grep -q 'destroy vm=41' "$calls"; then
printf 'successful job VM should remain idle until billing boundary\n' >&2
exit 1
fi
if grep -q 'destroy vm=42' "$calls"; then
printf 'failed bootstrapped reconciled VM should remain idle until billing boundary\n' >&2
exit 1
fi
jq -e 'select(.status == "idle_vm" and .idle_expires_at == 3600)' \
"$(gcr_record_path 101 1)" >/dev/null
jq -e 'select(.status == "idle_vm" and .idle_expires_at == 3601)' \
"$(gcr_record_path 102 1)" >/dev/null
gcr_idle_record_usable "$(gcr_record_get 102 1)"
idle_once="$(gcr_record_get 101 1)"
gcr_reap_finished_jobs
test "$(gcr_record_get 101 1)" = "$idle_once"
test ! -e "$(gcr_record_path 102 1)"
# Skipped terminal jobs follow same healthy retention policy without diagnostics.
record_skipped='{"job_id":"105","run_attempt":"1","repo":"hinterland/hearth","label":"nix","created_at":"1","ttl_min":480,"vm_id":45,"vm_name":"gcr-105-1","bootstrapped":true,"status":"vm_active"}'
gcr_record_put 105 1 "$record_skipped"
gcr_reap_finished_jobs
test "$(gcr_record_field "$(gcr_record_get 105 1)" status)" = idle_vm
if grep -q 'diag vm=45' "$calls" || grep -q 'destroy vm=45' "$calls"; then
printf 'healthy skipped VM must be retained without failure diagnostics\n' >&2
exit 1
fi
# Reaper keeps cleanup ownership after DELETE failure and retries next sweep.
record_delete_fail='{"job_id":"104","run_attempt":"1","repo":"hinterland/hearth","label":"nix","created_at":"1","ttl_min":480,"vm_id":44,"vm_name":"gcr-104-1","bootstrapped":true,"status":"vm_active"}'
@@ -76,6 +101,10 @@ gcr_vm_destroy() {
printf 'destroy-failed vm=%s\n' "$1" >> "$calls"
return 1
}
gcr_vm_runner_service() {
printf 'runner %s vm=%s\n' "$2" "$1" >> "$calls"
[ "$1" != 44 ] || [ "$2" != stop ]
}
gcr_reap_finished_jobs
test "$(gcr_record_field "$(gcr_record_get 104 1)" status)" = cleanup_pending
test "$(gcr_count_active)" = 1
@@ -101,6 +130,10 @@ gcr_gitea_job_state() {
gcr_vm_public_ip() {
return 1
}
gcr_vm_runner_service() {
printf 'runner %s vm=%s\n' "$2" "$1" >> "$calls"
return 1
}
gcr_vm_destroy() {
printf 'destroy vm=%s\n' "$1" >> "$calls"
@@ -109,5 +142,6 @@ gcr_vm_destroy() {
gcr_reap_finished_jobs
grep -q 'diag vm=43 ip= job=103 reason=completed:failure' "$calls_ip_fail"
grep -q 'runner health vm=43' "$calls_ip_fail"
grep -q 'destroy vm=43' "$calls_ip_fail"
test ! -e "$(gcr_record_path 103 1)"