hinterland/hearth
2
2
Fork 1
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: new diagnostics
runner nix smoke / nix label and flake smoke (push) Failing after 8s
Details

Browse Source
This commit is contained in:
yukkop
2026-09-10 15:31:13 +00:00
parent b3bcd6d5a8
commit df0638a0f7
8 changed files with 268 additions and 8 deletions
Download Patch File Download Diff File Expand all files Collapse all files
package/gitea-runner-controller/controller.sh
+16 -7
View File
@@ -33,6 +33,8 @@ gcr_sweep_ttl() {
vm_id="$(gcr_record_field "$rec" vm_id)"
gcr_log warn --ns=sweep "TTL exceeded job=$job_id age=${age}s max=${max_sec}s"
if [ -n "$vm_id" ] && [ "$vm_id" != "null" ] && [ "$vm_id" != "0" ]; then
ip="$(gcr_vm_public_ip "$vm_id" || true)"
gcr_vm_collect_diagnostics "$vm_id" "$ip" "$job_id" ttl || true
gcr_vm_destroy "$vm_id" || true
gcr_event "vm-destroyed" "$job_id" "{\"vm_id\":$vm_id,\"reason\":\"ttl\"}"
fi
@@ -166,13 +168,6 @@ $runners
EOF
}
gcr_vm_public_ip() {
# gcr_vm_public_ip SERVER_ID -> ipv4 or empty
if gcr_hcloud_req GET "/servers/$1"; then
jq -r '.server.public_net.ipv4.ip // ""' "$GCR_LAST_BODY"
fi
}
# Runs SSH-push bootstrap for VMs that were created but not yet provisioned.
# Registration token is fetched fresh per attempt (short-lived usefulness).
gcr_bootstrap_pending() {
@@ -193,6 +188,13 @@ gcr_bootstrap_pending() {
completed:*)
gcr_log info --ns=sweep "pending job=$job_id already terminal ($state), destroying vm=$vm_id"
if [ -n "$vm_id" ] && [ "$vm_id" != "0" ] && [ "$vm_id" != "null" ]; then
case "$state" in
completed:success|completed:cancelled|completed:skipped) ;;
*)
ip="$(gcr_vm_public_ip "$vm_id" || true)"
gcr_vm_collect_diagnostics "$vm_id" "$ip" "$job_id" "$state" || true
;;
esac
gcr_vm_destroy "$vm_id" || true
gcr_event "vm-destroyed" "$job_id" "{\"vm_id\":$vm_id,\"reason\":\"pending-job-completed\",\"state\":\"$state\"}"
fi
@@ -238,6 +240,13 @@ gcr_reap_finished_jobs() {
completed:*)
gcr_log info --ns=sweep "job=$job_id terminal ($state), destroying vm=$vm_id"
if [ -n "$vm_id" ] && [ "$vm_id" != "0" ] && [ "$vm_id" != "null" ]; then
case "$state" in
completed:success|completed:cancelled|completed:skipped) ;;
*)
ip="$(gcr_vm_public_ip "$vm_id" || true)"
gcr_vm_collect_diagnostics "$vm_id" "$ip" "$job_id" "$state" || true
;;
esac
gcr_vm_destroy "$vm_id" || true
gcr_event "vm-destroyed" "$job_id" "{\"vm_id\":$vm_id,\"reason\":\"job-completed\",\"state\":\"$state\"}"
fi
package/gitea-runner-controller/hcloud.sh
+49
View File
@@ -225,6 +225,55 @@ gcr_vm_destroy() {
fi
}
gcr_vm_public_ip() {
# gcr_vm_public_ip SERVER_ID -> ipv4 or empty
if gcr_hcloud_req GET "/servers/$1"; then
jq -r '.server.public_net.ipv4.ip // ""' "$GCR_LAST_BODY"
fi
}
gcr_vm_collect_diagnostics() {
vm_id="$1"; ip="$2"; job_id="$3"; reason="$4"
[ "${GCR_DESTROY_DIAGNOSTICS:-1}" = "1" ] || return 0
[ -n "$ip" ] || return 0
test -n "${GCR_SSH_PRIVKEY_FILE:-}" && test -r "$GCR_SSH_PRIVKEY_FILE" || {
gcr_log warn --ns=hcloud "skip diagnostics vm=$vm_id job=$job_id reason=$reason: SSH key unavailable"
return 0
}
key_tmp="$(mktemp "${TMPDIR:-/tmp}/gcr-diag-sshkey.XXXXXX")"
cat "$GCR_SSH_PRIVKEY_FILE" > "$key_tmp"
printf '\n' >> "$key_tmp"
chmod 0600 "$key_tmp"
timeout_sec="${GCR_DESTROY_DIAGNOSTICS_TIMEOUT_SEC:-20}"
ssh_opts="-i $key_tmp -o IdentitiesOnly=yes -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o ConnectTimeout=5 -o BatchMode=yes"
diag_out="$(mktemp "${TMPDIR:-/tmp}/gcr-diag-out.XXXXXX")"
gcr_log warn --ns=hcloud "pre-destroy diagnostics begin vm=$vm_id ip=$ip job=$job_id reason=$reason timeout=${timeout_sec}s"
if timeout -k 5 "$timeout_sec" ssh $ssh_opts "root@$ip" \
'set +e
export LC_ALL=C
printf "== time ==\n"; date -u
printf "== uptime ==\n"; uptime
printf "== memory ==\n"; free -h
printf "== disk ==\n"; df -h / /nix /var/lib 2>/dev/null || df -h
printf "== pressure ==\n"; cat /proc/pressure/cpu /proc/pressure/memory /proc/pressure/io 2>/dev/null
printf "== kernel failure signals ==\n"; dmesg -T 2>/dev/null | grep -Ei "out of memory|oom-kill|killed process|no space|I/O error|EXT4-fs error|xfs.*error|nvme.*error" | tail -n 80
printf "== runner service ==\n"; systemctl show gitea-runner.service -p ActiveState -p SubState -p Result -p ExecMainStatus -p ExecMainCode -p NRestarts 2>/dev/null
printf "== bootstrap service ==\n"; systemctl show gcr-bootstrap.service -p ActiveState -p SubState -p Result -p ExecMainStatus -p ExecMainCode 2>/dev/null
printf "== process sample ==\n"; ps -eo pid,ppid,stat,etime,comm 2>/dev/null | head -n 80' \
> "$diag_out" 2>&1; then
gcr_redact < "$diag_out" >&2
gcr_log warn --ns=hcloud "pre-destroy diagnostics complete vm=$vm_id job=$job_id reason=$reason"
else
gcr_redact < "$diag_out" >&2
gcr_log warn --ns=hcloud "pre-destroy diagnostics failed vm=$vm_id job=$job_id reason=$reason"
fi
rm -f "$key_tmp" "$diag_out"
return 0
}
# Bootstrap delivery is SSH-push from the controller. The MicroOS snapshot's
# cloud-init cannot fetch user-data (Hetzner datasource DHCP failure), so the
# controller drives provisioning over SSH using GCR_SSH_PRIVKEY_FILE, whose
package/gitea-runner-controller/webhook.sh
+9 -1
View File
@@ -175,6 +175,13 @@ gcr_deallocate() {
vm_id="$(gcr_record_field "$rec" vm_id)"
if [ -n "$vm_id" ] && [ "$vm_id" != "null" ] && [ "$vm_id" != "0" ]; then
case "$new_status" in
completed:success|completed:cancelled|completed:skipped) ;;
completed:*)
ip="$(gcr_vm_public_ip "$vm_id" || true)"
gcr_vm_collect_diagnostics "$vm_id" "$ip" "$job_id" "$new_status" || true
;;
esac
gcr_vm_destroy "$vm_id" || true
gcr_event "vm-destroyed" "$job_id" "{\"vm_id\":$vm_id,\"reason\":\"$new_status\"}"
fi
@@ -199,6 +206,7 @@ gcr_handle_webhook() {
action="$(printf '%s' "$gcr_body" | jq -r '.action // ""')"
job_id="$(printf '%s' "$gcr_body" | jq -r '.workflow_job.id // ""')"
attempt="$(printf '%s' "$gcr_body" | jq -r '.workflow_job.run_attempt // ""')"
conclusion="$(printf '%s' "$gcr_body" | jq -r '.workflow_job.conclusion // ""')"
repo="$(printf '%s' "$gcr_body" | jq -r '.repository.full_name // ""')"
labels_json="$(printf '%s' "$gcr_body" | jq -c '.workflow_job.labels // []')"
@@ -220,7 +228,7 @@ gcr_handle_webhook() {
RESPONSE_CODE=204
;;
completed)
gcr_deallocate "$job_id" "$attempt" "completed"
gcr_deallocate "$job_id" "$attempt" "completed:${conclusion:-unknown}"
RESPONSE_CODE=204
;;
*)