diff --git a/package/gitea-runner-controller/controller.sh b/package/gitea-runner-controller/controller.sh index cced855e..bfe03b5d 100644 --- a/package/gitea-runner-controller/controller.sh +++ b/package/gitea-runner-controller/controller.sh @@ -33,6 +33,8 @@ gcr_sweep_ttl() { vm_id="$(gcr_record_field "$rec" vm_id)" gcr_log warn --ns=sweep "TTL exceeded job=$job_id age=${age}s max=${max_sec}s" if [ -n "$vm_id" ] && [ "$vm_id" != "null" ] && [ "$vm_id" != "0" ]; then + ip="$(gcr_vm_public_ip "$vm_id" || true)" + gcr_vm_collect_diagnostics "$vm_id" "$ip" "$job_id" ttl || true gcr_vm_destroy "$vm_id" || true gcr_event "vm-destroyed" "$job_id" "{\"vm_id\":$vm_id,\"reason\":\"ttl\"}" fi @@ -166,13 +168,6 @@ $runners EOF } -gcr_vm_public_ip() { - # gcr_vm_public_ip SERVER_ID -> ipv4 or empty - if gcr_hcloud_req GET "/servers/$1"; then - jq -r '.server.public_net.ipv4.ip // ""' "$GCR_LAST_BODY" - fi -} - # Runs SSH-push bootstrap for VMs that were created but not yet provisioned. # Registration token is fetched fresh per attempt (short-lived usefulness). gcr_bootstrap_pending() { @@ -193,6 +188,13 @@ gcr_bootstrap_pending() { completed:*) gcr_log info --ns=sweep "pending job=$job_id already terminal ($state), destroying vm=$vm_id" if [ -n "$vm_id" ] && [ "$vm_id" != "0" ] && [ "$vm_id" != "null" ]; then + case "$state" in + completed:success|completed:cancelled|completed:skipped) ;; + *) + ip="$(gcr_vm_public_ip "$vm_id" || true)" + gcr_vm_collect_diagnostics "$vm_id" "$ip" "$job_id" "$state" || true + ;; + esac gcr_vm_destroy "$vm_id" || true gcr_event "vm-destroyed" "$job_id" "{\"vm_id\":$vm_id,\"reason\":\"pending-job-completed\",\"state\":\"$state\"}" fi @@ -238,6 +240,13 @@ gcr_reap_finished_jobs() { completed:*) gcr_log info --ns=sweep "job=$job_id terminal ($state), destroying vm=$vm_id" if [ -n "$vm_id" ] && [ "$vm_id" != "0" ] && [ "$vm_id" != "null" ]; then + case "$state" in + completed:success|completed:cancelled|completed:skipped) ;; + *) + ip="$(gcr_vm_public_ip "$vm_id" || true)" + gcr_vm_collect_diagnostics "$vm_id" "$ip" "$job_id" "$state" || true + ;; + esac gcr_vm_destroy "$vm_id" || true gcr_event "vm-destroyed" "$job_id" "{\"vm_id\":$vm_id,\"reason\":\"job-completed\",\"state\":\"$state\"}" fi diff --git a/package/gitea-runner-controller/hcloud.sh b/package/gitea-runner-controller/hcloud.sh index a03f4db6..2c070758 100644 --- a/package/gitea-runner-controller/hcloud.sh +++ b/package/gitea-runner-controller/hcloud.sh @@ -225,6 +225,55 @@ gcr_vm_destroy() { fi } +gcr_vm_public_ip() { + # gcr_vm_public_ip SERVER_ID -> ipv4 or empty + if gcr_hcloud_req GET "/servers/$1"; then + jq -r '.server.public_net.ipv4.ip // ""' "$GCR_LAST_BODY" + fi +} + +gcr_vm_collect_diagnostics() { + vm_id="$1"; ip="$2"; job_id="$3"; reason="$4" + + [ "${GCR_DESTROY_DIAGNOSTICS:-1}" = "1" ] || return 0 + [ -n "$ip" ] || return 0 + test -n "${GCR_SSH_PRIVKEY_FILE:-}" && test -r "$GCR_SSH_PRIVKEY_FILE" || { + gcr_log warn --ns=hcloud "skip diagnostics vm=$vm_id job=$job_id reason=$reason: SSH key unavailable" + return 0 + } + + key_tmp="$(mktemp "${TMPDIR:-/tmp}/gcr-diag-sshkey.XXXXXX")" + cat "$GCR_SSH_PRIVKEY_FILE" > "$key_tmp" + printf '\n' >> "$key_tmp" + chmod 0600 "$key_tmp" + timeout_sec="${GCR_DESTROY_DIAGNOSTICS_TIMEOUT_SEC:-20}" + ssh_opts="-i $key_tmp -o IdentitiesOnly=yes -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o ConnectTimeout=5 -o BatchMode=yes" + diag_out="$(mktemp "${TMPDIR:-/tmp}/gcr-diag-out.XXXXXX")" + + gcr_log warn --ns=hcloud "pre-destroy diagnostics begin vm=$vm_id ip=$ip job=$job_id reason=$reason timeout=${timeout_sec}s" + if timeout -k 5 "$timeout_sec" ssh $ssh_opts "root@$ip" \ + 'set +e + export LC_ALL=C + printf "== time ==\n"; date -u + printf "== uptime ==\n"; uptime + printf "== memory ==\n"; free -h + printf "== disk ==\n"; df -h / /nix /var/lib 2>/dev/null || df -h + printf "== pressure ==\n"; cat /proc/pressure/cpu /proc/pressure/memory /proc/pressure/io 2>/dev/null + printf "== kernel failure signals ==\n"; dmesg -T 2>/dev/null | grep -Ei "out of memory|oom-kill|killed process|no space|I/O error|EXT4-fs error|xfs.*error|nvme.*error" | tail -n 80 + printf "== runner service ==\n"; systemctl show gitea-runner.service -p ActiveState -p SubState -p Result -p ExecMainStatus -p ExecMainCode -p NRestarts 2>/dev/null + printf "== bootstrap service ==\n"; systemctl show gcr-bootstrap.service -p ActiveState -p SubState -p Result -p ExecMainStatus -p ExecMainCode 2>/dev/null + printf "== process sample ==\n"; ps -eo pid,ppid,stat,etime,comm 2>/dev/null | head -n 80' \ + > "$diag_out" 2>&1; then + gcr_redact < "$diag_out" >&2 + gcr_log warn --ns=hcloud "pre-destroy diagnostics complete vm=$vm_id job=$job_id reason=$reason" + else + gcr_redact < "$diag_out" >&2 + gcr_log warn --ns=hcloud "pre-destroy diagnostics failed vm=$vm_id job=$job_id reason=$reason" + fi + rm -f "$key_tmp" "$diag_out" + return 0 +} + # Bootstrap delivery is SSH-push from the controller. The MicroOS snapshot's # cloud-init cannot fetch user-data (Hetzner datasource DHCP failure), so the # controller drives provisioning over SSH using GCR_SSH_PRIVKEY_FILE, whose diff --git a/package/gitea-runner-controller/webhook.sh b/package/gitea-runner-controller/webhook.sh index 99c53041..0d862a2c 100644 --- a/package/gitea-runner-controller/webhook.sh +++ b/package/gitea-runner-controller/webhook.sh @@ -175,6 +175,13 @@ gcr_deallocate() { vm_id="$(gcr_record_field "$rec" vm_id)" if [ -n "$vm_id" ] && [ "$vm_id" != "null" ] && [ "$vm_id" != "0" ]; then + case "$new_status" in + completed:success|completed:cancelled|completed:skipped) ;; + completed:*) + ip="$(gcr_vm_public_ip "$vm_id" || true)" + gcr_vm_collect_diagnostics "$vm_id" "$ip" "$job_id" "$new_status" || true + ;; + esac gcr_vm_destroy "$vm_id" || true gcr_event "vm-destroyed" "$job_id" "{\"vm_id\":$vm_id,\"reason\":\"$new_status\"}" fi @@ -199,6 +206,7 @@ gcr_handle_webhook() { action="$(printf '%s' "$gcr_body" | jq -r '.action // ""')" job_id="$(printf '%s' "$gcr_body" | jq -r '.workflow_job.id // ""')" attempt="$(printf '%s' "$gcr_body" | jq -r '.workflow_job.run_attempt // ""')" + conclusion="$(printf '%s' "$gcr_body" | jq -r '.workflow_job.conclusion // ""')" repo="$(printf '%s' "$gcr_body" | jq -r '.repository.full_name // ""')" labels_json="$(printf '%s' "$gcr_body" | jq -c '.workflow_job.labels // []')" @@ -220,7 +228,7 @@ gcr_handle_webhook() { RESPONSE_CODE=204 ;; completed) - gcr_deallocate "$job_id" "$attempt" "completed" + gcr_deallocate "$job_id" "$attempt" "completed:${conclusion:-unknown}" RESPONSE_CODE=204 ;; *) diff --git a/test/package/default.nix b/test/package/default.nix index d94b1c93..05dd9387 100644 --- a/test/package/default.nix +++ b/test/package/default.nix @@ -2,6 +2,7 @@ (import ./migrator { inherit system inputs self pkgs; }) // (import ./hemar { inherit system inputs self pkgs; }) // (import (./. + "/sentinèlla") { inherit system inputs self pkgs; }) // + (import ./gitea-runner-controller { inherit system inputs self pkgs; }) // (import ./db-tool { inherit system inputs self pkgs; }) // (import ./with-attic-cache { inherit system inputs self pkgs; }) // (import ./element-web { inherit system inputs self pkgs; }) // diff --git a/test/package/gitea-runner-controller/default.nix b/test/package/gitea-runner-controller/default.nix new file mode 100644 index 00000000..eaefee62 --- /dev/null +++ b/test/package/gitea-runner-controller/default.nix @@ -0,0 +1,54 @@ +{ inputs, self, pkgs, system, ... }: +let + lib = inputs.nixpkgs.lib; + + mkTestDrv = name: type: + if type == "directory" then + pkgs.runCommand "test-${name}" {} '' + if ! [ -f ${./test + "/${name}" + /run.sh} ]; then + echo "no run.sh in test/${name}" + exit 1 + fi + mkdir -p "$out" + cp -r ${./test + "/${name}"}/* "$out/" + chmod +x "$out/run.sh" + '' + else if lib.hasSuffix ".sh" name then + pkgs.runCommand "test-${lib.removeSuffix ".sh" name}" {} '' + mkdir -p "$out" + install -Dm755 ${./test + "/${name}"} "$out/run.sh" + '' + else + null; + + testDir = builtins.readDir ./test; + testDrvs = + lib.mapAttrs' (n: v: + lib.nameValuePair (lib.removeSuffix ".sh" n) v + ) (lib.filterAttrs (_: v: v != null) + (lib.mapAttrs (n: t: mkTestDrv n t) testDir)); + + mkTest = testName: testDrv: pkgs.runCommand "gitea-runner-controller-test-${testName}" + { + nativeBuildInputs = [ + pkgs.coreutils + pkgs.dash + pkgs.gnugrep + pkgs.gnused + pkgs.jq + ]; + LOG_SH = ../../../package/gitea-runner-controller/log.sh; + STATE_SH = ../../../package/gitea-runner-controller/state.sh; + DECIDE_SH = ../../../package/gitea-runner-controller/decide.sh; + HCLOUD_SH = ../../../package/gitea-runner-controller/hcloud.sh; + GITEA_SH = ../../../package/gitea-runner-controller/gitea.sh; + CONTROLLER_SH = ../../../package/gitea-runner-controller/controller.sh; + WEBHOOK_SH = ../../../package/gitea-runner-controller/webhook.sh; + } '' + test=${testDrv} + ${builtins.readFile ./launch.sh} + mkdir -p "$out" + ''; +in lib.mapAttrs' (name: drv: + lib.nameValuePair "gitea-runner-controller-${name}" (mkTest name drv) +) testDrvs diff --git a/test/package/gitea-runner-controller/launch.sh b/test/package/gitea-runner-controller/launch.sh new file mode 100644 index 00000000..cb8e6289 --- /dev/null +++ b/test/package/gitea-runner-controller/launch.sh @@ -0,0 +1,9 @@ +#!/bin/dash +set -eu + +GCR_STATE_DIR="$(mktemp -d)" +export GCR_STATE_DIR +export GCR_LOG=error +trap 'rm -rf "$GCR_STATE_DIR"' EXIT INT HUP + +dash "$test/run.sh" diff --git a/test/package/gitea-runner-controller/test/reap-diagnostics.sh b/test/package/gitea-runner-controller/test/reap-diagnostics.sh new file mode 100644 index 00000000..ef546ef6 --- /dev/null +++ b/test/package/gitea-runner-controller/test/reap-diagnostics.sh @@ -0,0 +1,72 @@ +#!/bin/dash +set -eu + +. "$LOG_SH" +. "$STATE_SH" +. "$DECIDE_SH" +. "$HCLOUD_SH" +. "$GITEA_SH" +. "$CONTROLLER_SH" + +gcr_state_init +calls="$GCR_STATE_DIR/calls" + +gcr_gitea_job_state() { + case "$2" in + 101) printf 'completed:success' ;; + 102) printf 'completed:failure' ;; + *) return 1 ;; + esac +} + +gcr_vm_public_ip() { + printf '192.0.2.%s' "$1" +} + +gcr_vm_collect_diagnostics() { + printf 'diag vm=%s ip=%s job=%s reason=%s\n' "$1" "$2" "$3" "$4" >> "$calls" +} + +gcr_vm_destroy() { + printf 'destroy vm=%s\n' "$1" >> "$calls" +} + +record_success='{"job_id":"101","run_attempt":"1","repo":"hinterland/hearth","label":"gross-nix-x86-perf","created_at":"1","ttl_min":480,"vm_id":41,"vm_name":"gcr-101-1","bootstrapped":true,"status":"vm_active"}' +record_failure='{"job_id":"102","run_attempt":"1","repo":"hinterland/hearth","label":"gross-nix-x86-perf","created_at":"1","ttl_min":480,"vm_id":42,"vm_name":"gcr-102-1","bootstrapped":true,"status":"vm_active"}' + +gcr_record_put 101 1 "$record_success" +gcr_record_put 102 1 "$record_failure" +gcr_reap_finished_jobs + +grep -q 'destroy vm=41' "$calls" +grep -q 'destroy vm=42' "$calls" +grep -q 'diag vm=42 ip=192.0.2.42 job=102 reason=completed:failure' "$calls" +if grep -q 'diag vm=41' "$calls"; then + printf 'success job should not collect diagnostics\n' >&2 + exit 1 +fi + +test ! -e "$(gcr_record_path 101 1)" +test ! -e "$(gcr_record_path 102 1)" + +calls_ip_fail="$GCR_STATE_DIR/calls-ip-fail" +calls="$calls_ip_fail" +record_ip_fail='{"job_id":"103","run_attempt":"1","repo":"hinterland/hearth","label":"gross-nix-x86-perf","created_at":"1","ttl_min":480,"vm_id":43,"vm_name":"gcr-103-1","bootstrapped":true,"status":"vm_active"}' +gcr_record_put 103 1 "$record_ip_fail" + +gcr_gitea_job_state() { + case "$2" in + 103) printf 'completed:failure' ;; + *) return 1 ;; + esac +} + +gcr_vm_public_ip() { + return 1 +} + +gcr_reap_finished_jobs + +grep -q 'diag vm=43 ip= job=103 reason=completed:failure' "$calls_ip_fail" +grep -q 'destroy vm=43' "$calls_ip_fail" +test ! -e "$(gcr_record_path 103 1)" diff --git a/test/package/gitea-runner-controller/test/webhook-diagnostics.sh b/test/package/gitea-runner-controller/test/webhook-diagnostics.sh new file mode 100644 index 00000000..de06c7d5 --- /dev/null +++ b/test/package/gitea-runner-controller/test/webhook-diagnostics.sh @@ -0,0 +1,58 @@ +#!/bin/dash +set -eu + +. "$LOG_SH" +. "$STATE_SH" +. "$DECIDE_SH" +. "$HCLOUD_SH" +. "$GITEA_SH" +. "$WEBHOOK_SH" + +gcr_state_init +calls="$GCR_STATE_DIR/calls" + +gcr_vm_public_ip() { + printf '192.0.2.%s' "$1" +} + +gcr_vm_collect_diagnostics() { + printf 'diag vm=%s ip=%s job=%s reason=%s\n' "$1" "$2" "$3" "$4" >> "$calls" +} + +gcr_vm_destroy() { + printf 'destroy vm=%s\n' "$1" >> "$calls" +} + +record_success='{"job_id":"201","run_attempt":"1","repo":"hinterland/hearth","label":"gross-nix-x86-perf","created_at":"1","ttl_min":480,"vm_id":51,"vm_name":"gcr-201-1","bootstrapped":true,"status":"vm_active"}' +record_failure='{"job_id":"202","run_attempt":"1","repo":"hinterland/hearth","label":"gross-nix-x86-perf","created_at":"1","ttl_min":480,"vm_id":52,"vm_name":"gcr-202-1","bootstrapped":true,"status":"vm_active"}' + +gcr_record_put 201 1 "$record_success" +gcr_record_put 202 1 "$record_failure" +gcr_deallocate 201 1 completed:success +gcr_deallocate 202 1 completed:failure + +grep -q 'destroy vm=51' "$calls" +grep -q 'destroy vm=52' "$calls" +grep -q 'diag vm=52 ip=192.0.2.52 job=202 reason=completed:failure' "$calls" +if grep -q 'diag vm=51' "$calls"; then + printf 'success webhook should not collect diagnostics\n' >&2 + exit 1 +fi + +test ! -e "$(gcr_record_path 201 1)" +test ! -e "$(gcr_record_path 202 1)" + +calls_ip_fail="$GCR_STATE_DIR/calls-ip-fail" +calls="$calls_ip_fail" +record_ip_fail='{"job_id":"203","run_attempt":"1","repo":"hinterland/hearth","label":"gross-nix-x86-perf","created_at":"1","ttl_min":480,"vm_id":53,"vm_name":"gcr-203-1","bootstrapped":true,"status":"vm_active"}' +gcr_record_put 203 1 "$record_ip_fail" + +gcr_vm_public_ip() { + return 1 +} + +gcr_deallocate 203 1 completed:failure + +grep -q 'diag vm=53 ip= job=203 reason=completed:failure' "$calls_ip_fail" +grep -q 'destroy vm=53' "$calls_ip_fail" +test ! -e "$(gcr_record_path 203 1)"