hinterland/hearth
2
2
Fork 1
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: new diagnostics
runner nix smoke / nix label and flake smoke (push) Failing after 8s
Details

Browse Source
This commit is contained in:
yukkop
2026-09-10 15:31:13 +00:00
parent b3bcd6d5a8
commit df0638a0f7
8 changed files with 268 additions and 8 deletions
Download Patch File Download Diff File Expand all files Collapse all files
package/gitea-runner-controller/controller.sh
+16 -7
View File
@@ -33,6 +33,8 @@ gcr_sweep_ttl() {
vm_id="$(gcr_record_field "$rec" vm_id)"
gcr_log warn --ns=sweep "TTL exceeded job=$job_id age=${age}s max=${max_sec}s"
if [ -n "$vm_id" ] && [ "$vm_id" != "null" ] && [ "$vm_id" != "0" ]; then
ip="$(gcr_vm_public_ip "$vm_id" || true)"
gcr_vm_collect_diagnostics "$vm_id" "$ip" "$job_id" ttl || true
gcr_vm_destroy "$vm_id" || true
gcr_event "vm-destroyed" "$job_id" "{\"vm_id\":$vm_id,\"reason\":\"ttl\"}"
fi
@@ -166,13 +168,6 @@ $runners
EOF
}
gcr_vm_public_ip() {
# gcr_vm_public_ip SERVER_ID -> ipv4 or empty
if gcr_hcloud_req GET "/servers/$1"; then
jq -r '.server.public_net.ipv4.ip // ""' "$GCR_LAST_BODY"
fi
}
# Runs SSH-push bootstrap for VMs that were created but not yet provisioned.
# Registration token is fetched fresh per attempt (short-lived usefulness).
gcr_bootstrap_pending() {
@@ -193,6 +188,13 @@ gcr_bootstrap_pending() {
completed:*)
gcr_log info --ns=sweep "pending job=$job_id already terminal ($state), destroying vm=$vm_id"
if [ -n "$vm_id" ] && [ "$vm_id" != "0" ] && [ "$vm_id" != "null" ]; then
case "$state" in
completed:success|completed:cancelled|completed:skipped) ;;
*)
ip="$(gcr_vm_public_ip "$vm_id" || true)"
gcr_vm_collect_diagnostics "$vm_id" "$ip" "$job_id" "$state" || true
;;
esac
gcr_vm_destroy "$vm_id" || true
gcr_event "vm-destroyed" "$job_id" "{\"vm_id\":$vm_id,\"reason\":\"pending-job-completed\",\"state\":\"$state\"}"
fi
@@ -238,6 +240,13 @@ gcr_reap_finished_jobs() {
completed:*)
gcr_log info --ns=sweep "job=$job_id terminal ($state), destroying vm=$vm_id"
if [ -n "$vm_id" ] && [ "$vm_id" != "0" ] && [ "$vm_id" != "null" ]; then
case "$state" in
completed:success|completed:cancelled|completed:skipped) ;;
*)
ip="$(gcr_vm_public_ip "$vm_id" || true)"
gcr_vm_collect_diagnostics "$vm_id" "$ip" "$job_id" "$state" || true
;;
esac
gcr_vm_destroy "$vm_id" || true
gcr_event "vm-destroyed" "$job_id" "{\"vm_id\":$vm_id,\"reason\":\"job-completed\",\"state\":\"$state\"}"
fi
package/gitea-runner-controller/hcloud.sh
+49
View File
@@ -225,6 +225,55 @@ gcr_vm_destroy() {
fi
}
gcr_vm_public_ip() {
# gcr_vm_public_ip SERVER_ID -> ipv4 or empty
if gcr_hcloud_req GET "/servers/$1"; then
jq -r '.server.public_net.ipv4.ip // ""' "$GCR_LAST_BODY"
fi
}
gcr_vm_collect_diagnostics() {
vm_id="$1"; ip="$2"; job_id="$3"; reason="$4"
[ "${GCR_DESTROY_DIAGNOSTICS:-1}" = "1" ] || return 0
[ -n "$ip" ] || return 0
test -n "${GCR_SSH_PRIVKEY_FILE:-}" && test -r "$GCR_SSH_PRIVKEY_FILE" || {
gcr_log warn --ns=hcloud "skip diagnostics vm=$vm_id job=$job_id reason=$reason: SSH key unavailable"
return 0
}
key_tmp="$(mktemp "${TMPDIR:-/tmp}/gcr-diag-sshkey.XXXXXX")"
cat "$GCR_SSH_PRIVKEY_FILE" > "$key_tmp"
printf '\n' >> "$key_tmp"
chmod 0600 "$key_tmp"
timeout_sec="${GCR_DESTROY_DIAGNOSTICS_TIMEOUT_SEC:-20}"
ssh_opts="-i $key_tmp -o IdentitiesOnly=yes -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o ConnectTimeout=5 -o BatchMode=yes"
diag_out="$(mktemp "${TMPDIR:-/tmp}/gcr-diag-out.XXXXXX")"
gcr_log warn --ns=hcloud "pre-destroy diagnostics begin vm=$vm_id ip=$ip job=$job_id reason=$reason timeout=${timeout_sec}s"
if timeout -k 5 "$timeout_sec" ssh $ssh_opts "root@$ip" \
'set +e
export LC_ALL=C
printf "== time ==\n"; date -u
printf "== uptime ==\n"; uptime
printf "== memory ==\n"; free -h
printf "== disk ==\n"; df -h / /nix /var/lib 2>/dev/null || df -h
printf "== pressure ==\n"; cat /proc/pressure/cpu /proc/pressure/memory /proc/pressure/io 2>/dev/null
printf "== kernel failure signals ==\n"; dmesg -T 2>/dev/null | grep -Ei "out of memory|oom-kill|killed process|no space|I/O error|EXT4-fs error|xfs.*error|nvme.*error" | tail -n 80
printf "== runner service ==\n"; systemctl show gitea-runner.service -p ActiveState -p SubState -p Result -p ExecMainStatus -p ExecMainCode -p NRestarts 2>/dev/null
printf "== bootstrap service ==\n"; systemctl show gcr-bootstrap.service -p ActiveState -p SubState -p Result -p ExecMainStatus -p ExecMainCode 2>/dev/null
printf "== process sample ==\n"; ps -eo pid,ppid,stat,etime,comm 2>/dev/null | head -n 80' \
> "$diag_out" 2>&1; then
gcr_redact < "$diag_out" >&2
gcr_log warn --ns=hcloud "pre-destroy diagnostics complete vm=$vm_id job=$job_id reason=$reason"
else
gcr_redact < "$diag_out" >&2
gcr_log warn --ns=hcloud "pre-destroy diagnostics failed vm=$vm_id job=$job_id reason=$reason"
fi
rm -f "$key_tmp" "$diag_out"
return 0
}
# Bootstrap delivery is SSH-push from the controller. The MicroOS snapshot's
# cloud-init cannot fetch user-data (Hetzner datasource DHCP failure), so the
# controller drives provisioning over SSH using GCR_SSH_PRIVKEY_FILE, whose
package/gitea-runner-controller/webhook.sh
+9 -1
View File
@@ -175,6 +175,13 @@ gcr_deallocate() {
vm_id="$(gcr_record_field "$rec" vm_id)"
if [ -n "$vm_id" ] && [ "$vm_id" != "null" ] && [ "$vm_id" != "0" ]; then
case "$new_status" in
completed:success|completed:cancelled|completed:skipped) ;;
completed:*)
ip="$(gcr_vm_public_ip "$vm_id" || true)"
gcr_vm_collect_diagnostics "$vm_id" "$ip" "$job_id" "$new_status" || true
;;
esac
gcr_vm_destroy "$vm_id" || true
gcr_event "vm-destroyed" "$job_id" "{\"vm_id\":$vm_id,\"reason\":\"$new_status\"}"
fi
@@ -199,6 +206,7 @@ gcr_handle_webhook() {
action="$(printf '%s' "$gcr_body" | jq -r '.action // ""')"
job_id="$(printf '%s' "$gcr_body" | jq -r '.workflow_job.id // ""')"
attempt="$(printf '%s' "$gcr_body" | jq -r '.workflow_job.run_attempt // ""')"
conclusion="$(printf '%s' "$gcr_body" | jq -r '.workflow_job.conclusion // ""')"
repo="$(printf '%s' "$gcr_body" | jq -r '.repository.full_name // ""')"
labels_json="$(printf '%s' "$gcr_body" | jq -c '.workflow_job.labels // []')"
@@ -220,7 +228,7 @@ gcr_handle_webhook() {
RESPONSE_CODE=204
;;
completed)
gcr_deallocate "$job_id" "$attempt" "completed"
gcr_deallocate "$job_id" "$attempt" "completed:${conclusion:-unknown}"
RESPONSE_CODE=204
;;
*)
test/package/default.nix
+1
View File
@@ -2,6 +2,7 @@
(import ./migrator { inherit system inputs self pkgs; }) //
(import ./hemar { inherit system inputs self pkgs; }) //
(import (./. + "/sentinèlla") { inherit system inputs self pkgs; }) //
(import ./gitea-runner-controller { inherit system inputs self pkgs; }) //
(import ./db-tool { inherit system inputs self pkgs; }) //
(import ./with-attic-cache { inherit system inputs self pkgs; }) //
(import ./element-web { inherit system inputs self pkgs; }) //
test/package/gitea-runner-controller/default.nix
+54
View File
@@ -0,0 +1,54 @@
{ inputs, self, pkgs, system, ... }:
let
lib = inputs.nixpkgs.lib;
mkTestDrv = name: type:
if type == "directory" then
pkgs.runCommand "test-${name}" {} ''
if ! [ -f ${./test + "/${name}" + /run.sh} ]; then
echo "no run.sh in test/${name}"
exit 1
fi
mkdir -p "$out"
cp -r ${./test + "/${name}"}/* "$out/"
chmod +x "$out/run.sh"
''
else if lib.hasSuffix ".sh" name then
pkgs.runCommand "test-${lib.removeSuffix ".sh" name}" {} ''
mkdir -p "$out"
install -Dm755 ${./test + "/${name}"} "$out/run.sh"
''
else
null;
testDir = builtins.readDir ./test;
testDrvs =
lib.mapAttrs' (n: v:
lib.nameValuePair (lib.removeSuffix ".sh" n) v
) (lib.filterAttrs (_: v: v != null)
(lib.mapAttrs (n: t: mkTestDrv n t) testDir));
mkTest = testName: testDrv: pkgs.runCommand "gitea-runner-controller-test-${testName}"
{
nativeBuildInputs = [
pkgs.coreutils
pkgs.dash
pkgs.gnugrep
pkgs.gnused
pkgs.jq
];
LOG_SH = ../../../package/gitea-runner-controller/log.sh;
STATE_SH = ../../../package/gitea-runner-controller/state.sh;
DECIDE_SH = ../../../package/gitea-runner-controller/decide.sh;
HCLOUD_SH = ../../../package/gitea-runner-controller/hcloud.sh;
GITEA_SH = ../../../package/gitea-runner-controller/gitea.sh;
CONTROLLER_SH = ../../../package/gitea-runner-controller/controller.sh;
WEBHOOK_SH = ../../../package/gitea-runner-controller/webhook.sh;
} ''
test=${testDrv}
${builtins.readFile ./launch.sh}
mkdir -p "$out"
'';
in lib.mapAttrs' (name: drv:
lib.nameValuePair "gitea-runner-controller-${name}" (mkTest name drv)
) testDrvs
test/package/gitea-runner-controller/launch.sh
+9
View File
@@ -0,0 +1,9 @@
#!/bin/dash
set -eu
GCR_STATE_DIR="$(mktemp -d)"
export GCR_STATE_DIR
export GCR_LOG=error
trap 'rm -rf "$GCR_STATE_DIR"' EXIT INT HUP
dash "$test/run.sh"
test/package/gitea-runner-controller/test/reap-diagnostics.sh
+72
View File
@@ -0,0 +1,72 @@
#!/bin/dash
set -eu
. "$LOG_SH"
. "$STATE_SH"
. "$DECIDE_SH"
. "$HCLOUD_SH"
. "$GITEA_SH"
. "$CONTROLLER_SH"
gcr_state_init
calls="$GCR_STATE_DIR/calls"
gcr_gitea_job_state() {
case "$2" in
101) printf 'completed:success' ;;
102) printf 'completed:failure' ;;
*) return 1 ;;
esac
}
gcr_vm_public_ip() {
printf '192.0.2.%s' "$1"
}
gcr_vm_collect_diagnostics() {
printf 'diag vm=%s ip=%s job=%s reason=%s\n' "$1" "$2" "$3" "$4" >> "$calls"
}
gcr_vm_destroy() {
printf 'destroy vm=%s\n' "$1" >> "$calls"
}
record_success='{"job_id":"101","run_attempt":"1","repo":"hinterland/hearth","label":"gross-nix-x86-perf","created_at":"1","ttl_min":480,"vm_id":41,"vm_name":"gcr-101-1","bootstrapped":true,"status":"vm_active"}'
record_failure='{"job_id":"102","run_attempt":"1","repo":"hinterland/hearth","label":"gross-nix-x86-perf","created_at":"1","ttl_min":480,"vm_id":42,"vm_name":"gcr-102-1","bootstrapped":true,"status":"vm_active"}'
gcr_record_put 101 1 "$record_success"
gcr_record_put 102 1 "$record_failure"
gcr_reap_finished_jobs
grep -q 'destroy vm=41' "$calls"
grep -q 'destroy vm=42' "$calls"
grep -q 'diag vm=42 ip=192.0.2.42 job=102 reason=completed:failure' "$calls"
if grep -q 'diag vm=41' "$calls"; then
printf 'success job should not collect diagnostics\n' >&2
exit 1
fi
test ! -e "$(gcr_record_path 101 1)"
test ! -e "$(gcr_record_path 102 1)"
calls_ip_fail="$GCR_STATE_DIR/calls-ip-fail"
calls="$calls_ip_fail"
record_ip_fail='{"job_id":"103","run_attempt":"1","repo":"hinterland/hearth","label":"gross-nix-x86-perf","created_at":"1","ttl_min":480,"vm_id":43,"vm_name":"gcr-103-1","bootstrapped":true,"status":"vm_active"}'
gcr_record_put 103 1 "$record_ip_fail"
gcr_gitea_job_state() {
case "$2" in
103) printf 'completed:failure' ;;
*) return 1 ;;
esac
}
gcr_vm_public_ip() {
return 1
}
gcr_reap_finished_jobs
grep -q 'diag vm=43 ip= job=103 reason=completed:failure' "$calls_ip_fail"
grep -q 'destroy vm=43' "$calls_ip_fail"
test ! -e "$(gcr_record_path 103 1)"
test/package/gitea-runner-controller/test/webhook-diagnostics.sh
+58
View File
@@ -0,0 +1,58 @@
#!/bin/dash
set -eu
. "$LOG_SH"
. "$STATE_SH"
. "$DECIDE_SH"
. "$HCLOUD_SH"
. "$GITEA_SH"
. "$WEBHOOK_SH"
gcr_state_init
calls="$GCR_STATE_DIR/calls"
gcr_vm_public_ip() {
printf '192.0.2.%s' "$1"
}
gcr_vm_collect_diagnostics() {
printf 'diag vm=%s ip=%s job=%s reason=%s\n' "$1" "$2" "$3" "$4" >> "$calls"
}
gcr_vm_destroy() {
printf 'destroy vm=%s\n' "$1" >> "$calls"
}
record_success='{"job_id":"201","run_attempt":"1","repo":"hinterland/hearth","label":"gross-nix-x86-perf","created_at":"1","ttl_min":480,"vm_id":51,"vm_name":"gcr-201-1","bootstrapped":true,"status":"vm_active"}'
record_failure='{"job_id":"202","run_attempt":"1","repo":"hinterland/hearth","label":"gross-nix-x86-perf","created_at":"1","ttl_min":480,"vm_id":52,"vm_name":"gcr-202-1","bootstrapped":true,"status":"vm_active"}'
gcr_record_put 201 1 "$record_success"
gcr_record_put 202 1 "$record_failure"
gcr_deallocate 201 1 completed:success
gcr_deallocate 202 1 completed:failure
grep -q 'destroy vm=51' "$calls"
grep -q 'destroy vm=52' "$calls"
grep -q 'diag vm=52 ip=192.0.2.52 job=202 reason=completed:failure' "$calls"
if grep -q 'diag vm=51' "$calls"; then
printf 'success webhook should not collect diagnostics\n' >&2
exit 1
fi
test ! -e "$(gcr_record_path 201 1)"
test ! -e "$(gcr_record_path 202 1)"
calls_ip_fail="$GCR_STATE_DIR/calls-ip-fail"
calls="$calls_ip_fail"
record_ip_fail='{"job_id":"203","run_attempt":"1","repo":"hinterland/hearth","label":"gross-nix-x86-perf","created_at":"1","ttl_min":480,"vm_id":53,"vm_name":"gcr-203-1","bootstrapped":true,"status":"vm_active"}'
gcr_record_put 203 1 "$record_ip_fail"
gcr_vm_public_ip() {
return 1
}
gcr_deallocate 203 1 completed:failure
grep -q 'diag vm=53 ip= job=203 reason=completed:failure' "$calls_ip_fail"
grep -q 'destroy vm=53' "$calls_ip_fail"
test ! -e "$(gcr_record_path 203 1)"