diff --git a/package/gitea-runner-controller/hcloud.sh b/package/gitea-runner-controller/hcloud.sh index 0a6d76c5..ecc69095 100644 --- a/package/gitea-runner-controller/hcloud.sh +++ b/package/gitea-runner-controller/hcloud.sh @@ -409,7 +409,11 @@ gcr_vm_public_ip() { # The controller starts the service only after the claim record is written. gcr_vm_runner_service() { vm_id="$1"; action="$2" - case "$action" in start|stop) ;; *) return 1 ;; esac + case "$action" in + start|stop) service_command="systemctl $action gitea-runner.service" ;; + health) service_command="systemctl is-active --quiet gitea-runner.service" ;; + *) return 1 ;; + esac ip="$(gcr_vm_public_ip "$vm_id")" || return 1 [ -n "$ip" ] || return 1 test -n "${GCR_SSH_PRIVKEY_FILE:-}" && test -r "$GCR_SSH_PRIVKEY_FILE" || return 1 @@ -418,7 +422,7 @@ gcr_vm_runner_service() { printf '\n' >> "$key_tmp" chmod 0600 "$key_tmp" ssh_opts="-i $key_tmp -o IdentitiesOnly=yes -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o ConnectTimeout=5 -o BatchMode=yes" - if timeout 30 ssh $ssh_opts "root@$ip" "systemctl $action gitea-runner.service"; then + if timeout 30 ssh $ssh_opts "root@$ip" "$service_command"; then rm -f "$key_tmp" return 0 fi @@ -468,6 +472,76 @@ gcr_vm_collect_diagnostics() { return 0 } +# Finish a terminal job under its allocation lock. Healthy bootstrapped VMs +# become idle until their existing billing boundary; every unsafe transition +# uses durable cleanup_pending teardown instead. +gcr_vm_finish_terminal() { + finish_job="$1"; finish_attempt="$2"; finish_rec="$3"; finish_state="$4" + finish_via="${5:-webhook}" + finish_vm_id="$(gcr_record_field "$finish_rec" vm_id)" + + if [ -n "$finish_vm_id" ] && [ "$finish_vm_id" != "null" ] \ + && [ "$finish_vm_id" != "0" ]; then + case "$finish_state" in + completed:success|completed:cancelled|completed:skipped) ;; + completed:*) + finish_ip="$(gcr_vm_public_ip "$finish_vm_id" || true)" + gcr_vm_collect_diagnostics "$finish_vm_id" "$finish_ip" \ + "$finish_job" "$finish_state" || true + ;; + esac + fi + + if finish_idle_rec="$(gcr_record_idle_json "$finish_rec")"; then + gcr_lock_acquire idle-pool || return 2 + finish_repo="$(gcr_record_field "$finish_rec" repo)" + finish_runner="$(gcr_record_field "$finish_rec" vm_name)" + if ! gcr_vm_runner_service "$finish_vm_id" health; then + gcr_lock_release idle-pool + finish_cleanup_reason=idle-health-failed + elif ! gcr_gitea_runner_disabled "$finish_repo" "$finish_runner" true \ + || ! gcr_vm_runner_service "$finish_vm_id" stop; then + gcr_lock_release idle-pool + finish_cleanup_reason=idle-stop-failed + elif ! gcr_record_put "$finish_job" "$finish_attempt" "$finish_idle_rec"; then + gcr_lock_release idle-pool + finish_cleanup_reason=idle-state-write-failed + else + finish_expires="$(gcr_record_field "$finish_idle_rec" idle_expires_at)" + gcr_lock_release idle-pool + gcr_event "vm-idle" "$finish_job" \ + "{\"vm_id\":$finish_vm_id,\"expires_at\":$finish_expires,\"via\":\"$finish_via\"}" + gcr_log info --ns=sweep \ + "job=$finish_job terminal ($finish_state), retaining vm=$finish_vm_id until $finish_expires" + return 0 + fi + + if gcr_vm_cleanup_start "$finish_job" "$finish_attempt" "$finish_rec" \ + "$finish_cleanup_reason" false; then + gcr_event "vm-destroyed" "$finish_job" \ + "{\"vm_id\":$finish_vm_id,\"reason\":\"$finish_cleanup_reason\",\"via\":\"$finish_via\"}" + else + gcr_event "vm-cleanup-pending" "$finish_job" \ + "{\"vm_id\":$finish_vm_id,\"reason\":\"$finish_cleanup_reason\",\"via\":\"$finish_via\"}" + fi + return 0 + fi + + if [ -n "$finish_vm_id" ] && [ "$finish_vm_id" != "null" ] \ + && [ "$finish_vm_id" != "0" ]; then + if gcr_vm_cleanup_start "$finish_job" "$finish_attempt" "$finish_rec" \ + "$finish_state" false; then + gcr_event "vm-destroyed" "$finish_job" \ + "{\"vm_id\":$finish_vm_id,\"reason\":\"$finish_state\",\"via\":\"$finish_via\"}" + else + gcr_event "vm-cleanup-pending" "$finish_job" \ + "{\"vm_id\":$finish_vm_id,\"reason\":\"$finish_state\",\"via\":\"$finish_via\"}" + fi + else + gcr_record_del "$finish_job" "$finish_attempt" + fi +} + # Bootstrap delivery is SSH-push from the controller. The MicroOS snapshot's # cloud-init cannot fetch user-data (Hetzner datasource DHCP failure), so the # controller drives provisioning over SSH using GCR_SSH_PRIVKEY_FILE, whose diff --git a/package/gitea-runner-controller/state.sh b/package/gitea-runner-controller/state.sh index 65d82c44..72722798 100644 --- a/package/gitea-runner-controller/state.sh +++ b/package/gitea-runner-controller/state.sh @@ -98,8 +98,8 @@ gcr_now_epoch() { date -u '+%s' } -# Successful VMs remain reusable until next billing-hour boundary, but never -# beyond profile hard TTL. Prints updated idle record when retention is safe. +# Healthy bootstrapped VMs remain reusable until next billing-hour boundary, +# but never beyond profile hard TTL. Prints updated idle record when safe. gcr_record_idle_json() { gcr_idle_rec="$1" [ "$(gcr_record_field "$gcr_idle_rec" bootstrapped)" = "true" ] || return 1 diff --git a/test/package/gitea-runner-controller/test/idle-reuse.sh b/test/package/gitea-runner-controller/test/idle-reuse.sh index a2d8adcb..f3a1ccc7 100644 --- a/test/package/gitea-runner-controller/test/idle-reuse.sh +++ b/test/package/gitea-runner-controller/test/idle-reuse.sh @@ -67,21 +67,28 @@ jq -e 'select(.job_id == "302" and .vm_id == 71 and "$(gcr_record_path 302 1)" >/dev/null test "$(grep -Ec '^(budget|token|create)$' "$calls" || true)" = 0 -# Start failure keeps Gitea runner disabled and record retryable. +# Failed post-start health check keeps runner disabled and record retryable. retry_idle='{"job_id":"315","run_attempt":"1","repo":"hinterland/hearth","label":"gross-arm","created_at":1000,"ttl_min":180,"vm_id":79,"vm_name":"gcr-315-1","bootstrapped":true,"status":"idle_vm","idle_since":2000,"idle_expires_at":4600}' gcr_record_put 315 1 "$retry_idle" export GCR_PER_REPO_CAP=2 -FAIL_START=1 +FAIL_HEALTH=1 gcr_vm_runner_service() { printf 'runner %s vm=%s\n' "$2" "$1" >> "$calls" - [ "$2" = start ] && [ "$FAIL_START" = 1 ] && return 1 + [ "$2" = health ] && [ "$FAIL_HEALTH" = 1 ] && return 1 + return 0 } gcr_alloc 316 1 hinterland/hearth '["gross-arm"]' retry_rec="$(gcr_record_get 316 1)" test "$(gcr_record_field "$retry_rec" bootstrapped)" = false test "$(gcr_record_field "$retry_rec" reused_vm)" = true +grep -q '^runner start vm=79$' "$calls" +grep -q '^runner health vm=79$' "$calls" grep -q '^runner-disabled gcr-315-1 true$' "$calls" -FAIL_START=0 +if grep -q '^runner-disabled gcr-315-1 false$' "$calls"; then + printf 'unhealthy reused runner must never become schedulable\n' >&2 + exit 1 +fi +FAIL_HEALTH=0 gcr_record_del 316 1 # Expired idle capacity is never claimed; normal allocation then charges once.