fix: check runner health before reuse

package/gitea-runner-controller/hcloud.sh

@@ -409,7 +409,11 @@ gcr_vm_public_ip() {
 # The controller starts the service only after the claim record is written.
 gcr_vm_runner_service() {
     vm_id="$1"; action="$2"
-    case "$action" in start|stop) ;; *) return 1 ;; esac
+    case "$action" in
+        start|stop) service_command="systemctl $action gitea-runner.service" ;;
+        health) service_command="systemctl is-active --quiet gitea-runner.service" ;;
+        *) return 1 ;;
+    esac
     ip="$(gcr_vm_public_ip "$vm_id")" || return 1
     [ -n "$ip" ] || return 1
     test -n "${GCR_SSH_PRIVKEY_FILE:-}" && test -r "$GCR_SSH_PRIVKEY_FILE" || return 1
@@ -418,7 +422,7 @@ gcr_vm_runner_service() {
     printf '\n' >> "$key_tmp"
     chmod 0600 "$key_tmp"
     ssh_opts="-i $key_tmp -o IdentitiesOnly=yes -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o ConnectTimeout=5 -o BatchMode=yes"
-    if timeout 30 ssh $ssh_opts "root@$ip" "systemctl $action gitea-runner.service"; then
+    if timeout 30 ssh $ssh_opts "root@$ip" "$service_command"; then
         rm -f "$key_tmp"
         return 0
     fi
@@ -468,6 +472,76 @@ gcr_vm_collect_diagnostics() {
     return 0
 }
 
+# Finish a terminal job under its allocation lock. Healthy bootstrapped VMs
+# become idle until their existing billing boundary; every unsafe transition
+# uses durable cleanup_pending teardown instead.
+gcr_vm_finish_terminal() {
+    finish_job="$1"; finish_attempt="$2"; finish_rec="$3"; finish_state="$4"
+    finish_via="${5:-webhook}"
+    finish_vm_id="$(gcr_record_field "$finish_rec" vm_id)"
+
+    if [ -n "$finish_vm_id" ] && [ "$finish_vm_id" != "null" ] \
+        && [ "$finish_vm_id" != "0" ]; then
+        case "$finish_state" in
+            completed:success|completed:cancelled|completed:skipped) ;;
+            completed:*)
+                finish_ip="$(gcr_vm_public_ip "$finish_vm_id" || true)"
+                gcr_vm_collect_diagnostics "$finish_vm_id" "$finish_ip" \
+                    "$finish_job" "$finish_state" || true
+                ;;
+        esac
+    fi
+
+    if finish_idle_rec="$(gcr_record_idle_json "$finish_rec")"; then
+        gcr_lock_acquire idle-pool || return 2
+        finish_repo="$(gcr_record_field "$finish_rec" repo)"
+        finish_runner="$(gcr_record_field "$finish_rec" vm_name)"
+        if ! gcr_vm_runner_service "$finish_vm_id" health; then
+            gcr_lock_release idle-pool
+            finish_cleanup_reason=idle-health-failed
+        elif ! gcr_gitea_runner_disabled "$finish_repo" "$finish_runner" true \
+            || ! gcr_vm_runner_service "$finish_vm_id" stop; then
+            gcr_lock_release idle-pool
+            finish_cleanup_reason=idle-stop-failed
+        elif ! gcr_record_put "$finish_job" "$finish_attempt" "$finish_idle_rec"; then
+            gcr_lock_release idle-pool
+            finish_cleanup_reason=idle-state-write-failed
+        else
+            finish_expires="$(gcr_record_field "$finish_idle_rec" idle_expires_at)"
+            gcr_lock_release idle-pool
+            gcr_event "vm-idle" "$finish_job" \
+                "{\"vm_id\":$finish_vm_id,\"expires_at\":$finish_expires,\"via\":\"$finish_via\"}"
+            gcr_log info --ns=sweep \
+                "job=$finish_job terminal ($finish_state), retaining vm=$finish_vm_id until $finish_expires"
+            return 0
+        fi
+
+        if gcr_vm_cleanup_start "$finish_job" "$finish_attempt" "$finish_rec" \
+            "$finish_cleanup_reason" false; then
+            gcr_event "vm-destroyed" "$finish_job" \
+                "{\"vm_id\":$finish_vm_id,\"reason\":\"$finish_cleanup_reason\",\"via\":\"$finish_via\"}"
+        else
+            gcr_event "vm-cleanup-pending" "$finish_job" \
+                "{\"vm_id\":$finish_vm_id,\"reason\":\"$finish_cleanup_reason\",\"via\":\"$finish_via\"}"
+        fi
+        return 0
+    fi
+
+    if [ -n "$finish_vm_id" ] && [ "$finish_vm_id" != "null" ] \
+        && [ "$finish_vm_id" != "0" ]; then
+        if gcr_vm_cleanup_start "$finish_job" "$finish_attempt" "$finish_rec" \
+            "$finish_state" false; then
+            gcr_event "vm-destroyed" "$finish_job" \
+                "{\"vm_id\":$finish_vm_id,\"reason\":\"$finish_state\",\"via\":\"$finish_via\"}"
+        else
+            gcr_event "vm-cleanup-pending" "$finish_job" \
+                "{\"vm_id\":$finish_vm_id,\"reason\":\"$finish_state\",\"via\":\"$finish_via\"}"
+        fi
+    else
+        gcr_record_del "$finish_job" "$finish_attempt"
+    fi
+}
+
 # Bootstrap delivery is SSH-push from the controller. The MicroOS snapshot's
 # cloud-init cannot fetch user-data (Hetzner datasource DHCP failure), so the
 # controller drives provisioning over SSH using GCR_SSH_PRIVKEY_FILE, whose

package/gitea-runner-controller/state.sh

@@ -98,8 +98,8 @@ gcr_now_epoch() {
     date -u '+%s'
 }
 
-# Successful VMs remain reusable until next billing-hour boundary, but never
+# Healthy bootstrapped VMs remain reusable until next billing-hour boundary,
-# beyond profile hard TTL. Prints updated idle record when retention is safe.
+# but never beyond profile hard TTL. Prints updated idle record when safe.
 gcr_record_idle_json() {
     gcr_idle_rec="$1"
     [ "$(gcr_record_field "$gcr_idle_rec" bootstrapped)" = "true" ] || return 1

test/package/gitea-runner-controller/test/idle-reuse.sh

@@ -67,21 +67,28 @@ jq -e 'select(.job_id == "302" and .vm_id == 71 and
   "$(gcr_record_path 302 1)" >/dev/null
 test "$(grep -Ec '^(budget|token|create)$' "$calls" || true)" = 0
 
-# Start failure keeps Gitea runner disabled and record retryable.
+# Failed post-start health check keeps runner disabled and record retryable.
 retry_idle='{"job_id":"315","run_attempt":"1","repo":"hinterland/hearth","label":"gross-arm","created_at":1000,"ttl_min":180,"vm_id":79,"vm_name":"gcr-315-1","bootstrapped":true,"status":"idle_vm","idle_since":2000,"idle_expires_at":4600}'
 gcr_record_put 315 1 "$retry_idle"
 export GCR_PER_REPO_CAP=2
-FAIL_START=1
+FAIL_HEALTH=1
 gcr_vm_runner_service() {
   printf 'runner %s vm=%s\n' "$2" "$1" >> "$calls"
-  [ "$2" = start ] && [ "$FAIL_START" = 1 ] && return 1
+  [ "$2" = health ] && [ "$FAIL_HEALTH" = 1 ] && return 1
+  return 0
 }
 gcr_alloc 316 1 hinterland/hearth '["gross-arm"]'
 retry_rec="$(gcr_record_get 316 1)"
 test "$(gcr_record_field "$retry_rec" bootstrapped)" = false
 test "$(gcr_record_field "$retry_rec" reused_vm)" = true
+grep -q '^runner start vm=79$' "$calls"
+grep -q '^runner health vm=79$' "$calls"
 grep -q '^runner-disabled gcr-315-1 true$' "$calls"
-FAIL_START=0
+if grep -q '^runner-disabled gcr-315-1 false$' "$calls"; then
+  printf 'unhealthy reused runner must never become schedulable\n' >&2
+  exit 1
+fi
+FAIL_HEALTH=0
 gcr_record_del 316 1
 
 # Expired idle capacity is never claimed; normal allocation then charges once.